[
https://issues.apache.org/jira/browse/ATLAS-1572?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Vimal Sharma reassigned ATLAS-1572:
-----------------------------------
Assignee: Vimal Sharma
> sqoop metadata ingest fails with
> "org.apache.atlas.exception.AtlasBaseException: expected type List or Set;
> found org.apache.atlas.typesystem.Referenceable"
> ------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: ATLAS-1572
> URL: https://issues.apache.org/jira/browse/ATLAS-1572
> Project: Atlas
> Issue Type: Bug
> Components: atlas-intg
> Affects Versions: trunk, 0.8-incubating
> Reporter: Ayub Khan
> Assignee: Vimal Sharma
> Priority: Blocker
> Fix For: trunk, 0.8-incubating
>
> Attachments: ATLAS-1572.patch
>
>
> Sqoop import succeeds but the metadata ingest fails with below error
> {noformat}
> 2017-02-20 11:20:43,179 WARN - [NotificationHookConsumer thread-0:] ~ Max
> retries exceeded for message [{Id='(type: sqoop_dbdatastore, id:
> <unassigned>)', traits=[], values={owner=hrt_qa,
> storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
> dbStoreType=mysql, qualifiedName=mysql --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
> --table createSqoopTablejmnwwf8rji, name=mysql --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
> --table createSqoopTablejmnwwf8rji, description=,
> source=createSqoopTablejmnwwf8rji, storeUse=TABLE}}, {Id='(type: hive_db, id:
> <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
> clusterName=cl1}}, {Id='(type: hive_table, id: <unassigned>)', traits=[],
> values={name=createsqooptablejmnwwf8rji,
> qualifiedName=default.createsqooptablejmnwwf8rji@cl1, db={Id='(type: hive_db,
> id: <unassigned>)', traits=[], values={name=default,
> qualifiedName=default@cl1, clusterName=cl1}}}}, {Id='(type: sqoop_process,
> id: <unassigned>)', traits=[], values={outputs={Id='(type: hive_table, id:
> <unassigned>)', traits=[], values={name=createsqooptablejmnwwf8rji,
> qualifiedName=default.createsqooptablejmnwwf8rji@cl1, db={Id='(type: hive_db,
> id: <unassigned>)', traits=[], values={name=default,
> qualifiedName=default@cl1, clusterName=cl1}}}},
> commandlineOpts={db.clear.staging.table=false, hive.import=true,
> db.require.password=true, codegen.output.delimiters.enclose=0,
> codegen.input.delimiters.field=0, customtool.options.jsonmap={},
> hive.compute.stats.table=false,
> db.connect.string=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
> incremental.mode=None, db.table=createSqoopTablejmnwwf8rji, verbose=false,
> codegen.output.delimiters.enclose.required=false, mapreduce.num.mappers=4,
> hdfs.append.dir=false, direct.import=false, hive.drop.delims=false,
> hive.overwrite.table=false, hbase.bulk.load.enabled=false,
> hive.fail.table.exists=false, relaxed.isolation=false,
> hdfs.delete-target.dir=false, split.limit=null, db.username=sqoop,
> codegen.input.delimiters.enclose.required=false, codegen.output.dir=.,
> import.direct.split.size=0, reset.onemapper=false,
> codegen.output.delimiters.record=10, temporary.dirRoot=_sqoop,
> hcatalog.create.table=false, db.batch=false, import.fetch.size=-2147483648,
> accumulo.max.latency=5000, hdfs.file.format=TextFile,
> codegen.output.delimiters.field=1, mainframe.input.dataset.type=p,
> codegen.output.delimiters.escape=0, hcatalog.drop.and.create.table=false,
> import.max.inline.lob.size=16777216, hbase.create.table=false,
> codegen.auto.compile.dir=true,
> codegen.compile.dir=/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef,
> codegen.input.delimiters.enclose=0, export.new.update=UpdateOnly,
> enable.compression=false, skip.dist.cache=false,
> accumulo.batch.size=10240000, hdfs.warehouse.dir=/user/hrt_qa,
> codegen.input.delimiters.record=0, codegen.input.delimiters.escape=0,
> accumulo.create.table=false}, inputs={Id='(type: sqoop_dbdatastore, id:
> <unassigned>)', traits=[], values={owner=hrt_qa,
> storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
> dbStoreType=mysql, qualifiedName=mysql --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
> --table createSqoopTablejmnwwf8rji, name=mysql --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
> --table createSqoopTablejmnwwf8rji, description=,
> source=createSqoopTablejmnwwf8rji, storeUse=TABLE}}, qualifiedName=sqoop
> import --connect
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
> --table createSqoopTablejmnwwf8rji --hive-import --hive-database default
> --hive-table createsqooptablejmnwwf8rji --hive-cluster cl1, name=sqoop import
> --connect
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
> --table createSqoopTablejmnwwf8rji --hive-import --hive-database default
> --hive-table createsqooptablejmnwwf8rji --hive-cluster cl1,
> startTime=2017-02-20T11:20:04.901Z, endTime=2017-02-20T11:20:40.055Z,
> userName=hrt_qa, operation=import}}] (NotificationHookConsumer:324)
> org.apache.atlas.exception.AtlasBaseException: expected type List or Set;
> found org.apache.atlas.typesystem.Referenceable
> at
> org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:50)
> at
> org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:34)
> at
> org.apache.atlas.repository.converters.AtlasStructFormatConverter.fromV1ToV2(AtlasStructFormatConverter.java:169)
> at
> org.apache.atlas.repository.converters.AtlasEntityFormatConverter.fromV1ToV2(AtlasEntityFormatConverter.java:74)
> at
> org.apache.atlas.repository.converters.AtlasInstanceConverter.fromV1toV2Entity(AtlasInstanceConverter.java:222)
> at
> org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasEntities(AtlasInstanceConverter.java:204)
> at
> org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.handleMessage(NotificationHookConsumer.java:256)
> at
> org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.run(NotificationHookConsumer.java:230)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> 2017-02-20 11:21:06,094 ERROR - [NotificationHookConsumer thread-0:] ~
> [DROPPED_NOTIFICATION]
> {"version":{"version":"1.0.0"},"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
> --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
> --table
> createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
> --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
> --table
> createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888059","version":0,"typeName":"sqoop_process","state":"ACTIVE"},"typeName":"sqoop_process","values":{"name":"sqoop
> import --connect
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
> --table createSqoopTablejmnwwf8rji --hive-import --hive-database default
> --hive-table createsqooptablejmnwwf8rji --hive-cluster
> cl1","startTime":"2017-02-20T11:20:04.901Z","outputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},"commandlineOpts":{"db.clear.staging.table":"false","hive.import":"true","db.require.password":"true","codegen.output.delimiters.enclose":"0","codegen.input.delimiters.field":"0","customtool.options.jsonmap":"{}","hive.compute.stats.table":"false","db.connect.string":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","incremental.mode":"None","db.table":"createSqoopTablejmnwwf8rji","verbose":"false","codegen.output.delimiters.enclose.required":"false","mapreduce.num.mappers":"4","hdfs.append.dir":"false","direct.import":"false","hive.drop.delims":"false","hive.overwrite.table":"false","hbase.bulk.load.enabled":"false","hive.fail.table.exists":"false","relaxed.isolation":"false","hdfs.delete-target.dir":"false","split.limit":"null","db.username":"sqoop","codegen.input.delimiters.enclose.required":"false","codegen.output.dir":".","import.direct.split.size":"0","reset.onemapper":"false","codegen.output.delimiters.record":"10","temporary.dirRoot":"_sqoop","hcatalog.create.table":"false","db.batch":"false","import.fetch.size":"-2147483648","accumulo.max.latency":"5000","hdfs.file.format":"TextFile","codegen.output.delimiters.field":"1","mainframe.input.dataset.type":"p","codegen.output.delimiters.escape":"0","hcatalog.drop.and.create.table":"false","import.max.inline.lob.size":"16777216","hbase.create.table":"false","codegen.auto.compile.dir":"true","codegen.compile.dir":"/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef","codegen.input.delimiters.enclose":"0","export.new.update":"UpdateOnly","enable.compression":"false","skip.dist.cache":"false","accumulo.batch.size":"10240000","hdfs.warehouse.dir":"/user/hrt_qa","codegen.input.delimiters.record":"0","codegen.input.delimiters.escape":"0","accumulo.create.table":"false"},"endTime":"2017-02-20T11:20:40.055Z","inputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
> --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
> --table
> createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
> --url
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
> --table
> createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},"operation":"import","qualifiedName":"sqoop
> import --connect
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
> --table createSqoopTablejmnwwf8rji --hive-import --hive-database default
> --hive-table createsqooptablejmnwwf8rji --hive-cluster
> cl1","userName":"hrt_qa"},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_CREATE","user":"hrt_qa"}}
> (FAILED:339)
> {noformat}
> Steps to repro:
> Run the below sqoop import command
> {noformat}
> /usr/hdp/current/sqoop-client/bin/sqoop import --connect
> jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests
> --username sqoop --password sqoop --hive-import --warehouse-dir /user/hrt_qa
> --table createSqoopTablejmnwwf8rji
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
