elastic/elasticsearch

Getting i_o_exception BigInteger on documents creation/updation

Open

#44,150 opened on Jul 10, 2019

View on GitHub
 (15 comments) (0 reactions) (0 assignees)Java (76,700 stars) (25,882 forks)batch import
:Core/Infra/CoreTeam:Core/Infrahelp wantedtriaged

Description

Elasticsearch 7.2, java 1.8

I'm consistently getting this message in elastic logs:

[2019-07-09T14:15:18,836][WARN ][o.e.t.OutboundHandler    ] [banana] send message failed [channel: Netty4TcpChannel{localAddress=/***:55054, remoteAddress=****/****:9300}]
java.io.IOException: can not write type [class java.math.BigInteger]
	at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:758) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.io.stream.StreamOutput.lambda$static$11(StreamOutput.java:658) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:756) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.io.stream.StreamOutput.lambda$static$11(StreamOutput.java:658) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:756) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.io.stream.StreamOutput.writeMap(StreamOutput.java:511) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.script.Script.writeTo(Script.java:470) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.update.UpdateRequest.writeTo(UpdateRequest.java:891) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.DocWriteRequest.writeDocumentRequest(DocWriteRequest.java:248) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.bulk.BulkItemRequest.writeTo(BulkItemRequest.java:110) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.bulk.BulkShardRequest.writeTo(BulkShardRequest.java:80) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundMessage.writeMessage(OutboundMessage.java:70) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundMessage.serialize(OutboundMessage.java:53) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundHandler$MessageSerializer.get(OutboundHandler.java:169) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundHandler$MessageSerializer.get(OutboundHandler.java:155) ~[elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundHandler$SendContext.get(OutboundHandler.java:202) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundHandler.internalSend(OutboundHandler.java:132) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundHandler.sendMessage(OutboundHandler.java:127) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.OutboundHandler.sendRequest(OutboundHandler.java:91) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:246) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:643) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$1.sendRequest(SecurityServerTransportInterceptor.java:137) [x-pack-security-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:553) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:541) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performAction(TransportReplicationAction.java:749) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.performRemoteAction(TransportReplicationAction.java:723) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:689) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.doExecute(TransportReplicationAction.java:165) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.replication.TransportReplicationAction.doExecute(TransportReplicationAction.java:97) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:145) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:123) [x-pack-security-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:143) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:121) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:64) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation.doRun(TransportBulkAction.java:435) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.bulk.TransportBulkAction.executeBulk(TransportBulkAction.java:548) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.bulk.TransportBulkAction.doExecute(TransportBulkAction.java:253) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.bulk.TransportBulkAction.doExecute(TransportBulkAction.java:93) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:145) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:123) [x-pack-security-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:143) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:121) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:64) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:394) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.client.support.AbstractClient.bulk(AbstractClient.java:472) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.rest.action.document.RestBulkAction.lambda$prepareRequest$0(RestBulkAction.java:100) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:113) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.xpack.security.rest.SecurityRestFilter.handleRequest(SecurityRestFilter.java:69) [x-pack-security-7.2.0.jar:7.2.0]
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:337) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:320) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:370) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:299) [elasticsearch-7.2.0.jar:7.2.0]
	at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:66) [transport-netty4-client-7.2.0.jar:7.2.0]
	at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:31) [transport-netty4-client-7.2.0.jar:7.2.0]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:58) [transport-netty4-client-7.2.0.jar:7.2.0]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287) [netty-handler-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:682) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:582) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:536) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:906) [netty-common-4.1.35.Final.jar:4.1.35.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.35.Final.jar:4.1.35.Final]
	at java.lang.Thread.run(Thread.java:835) [?:?]

no more debug information on what exact data it happened

On the client side, I'm sending bulk request with dozens (sometimes hundreds) index/update requests, and some of them are returning with such an error: {"type":"i_o_exception","reason":"can not write type [class java.math.BigInteger]"} It's the only error information that is returned.

By the index of the errored response I could find the document that caused this error. Document has no big numbers at all.

It should be something internal? I've tried to send extremely big number and the error message looks like a normal validation exception - showing the problem field, etc. So, this case is other issue.

Contributor guide