[
https://issues.apache.org/jira/browse/MAPREDUCE-7372?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17497175#comment-17497175
]
Zhang Dongsheng commented on MAPREDUCE-7372:
--------------------------------------------
error log such as this.
{code:java}
2022-02-24 02:40:02,102 INFO tech.TestMapredTask: AnaSpeedMr
startingzds2022-02-24 02:40:02,815 INFO client.RMProxy: Connecting to
ResourceManager at nn/172.20.1.1:80322022-02-24 02:40:02,929 INFO
client.AHSProxy: Connecting to Application History server at
/0.0.0.0:102002022-02-24 02:40:02,950 INFO client.RMProxy: Connecting to
ResourceManager at nn/172.20.1.1:80322022-02-24 02:40:02,951 INFO
client.AHSProxy: Connecting to Application History server at
/0.0.0.0:102002022-02-24 02:40:03,029 WARN mapreduce.JobResourceUploader:
Hadoop command-line option parsing not performed. Implement the Tool interface
and execute your application with ToolRunner to remedy this.2022-02-24
02:40:03,041 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for
path: /tmp/zds/.staging/job_1645669537085_00022022-02-24 02:40:03,765 INFO
mapreduce.JobSubmitter: Cleaning up the staging area
/tmp/zds/.staging/job_1645669537085_0002Exception in thread "main"
org.apache.hadoop.security.AccessControlException: Permission denied: user=zds,
access=WRITE,
inode="/tmp/zds/.staging/job_1645669537085_0002/job.jar":zds:supergroup:----rw-rw-
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:399)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:261)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:193)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1896)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1880)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPathAccess(FSDirectory.java:1830)
at
org.apache.hadoop.hdfs.server.namenode.FSDirAttrOp.setReplication(FSDirAttrOp.java:136)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.setReplication(FSNamesystem.java:2237)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.setReplication(NameNodeRpcServer.java:856)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.setReplication(ClientNamenodeProtocolServerSideTranslatorPB.java:524)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1086) at
org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1029) at
org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:957) at
java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2957)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method) at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88)
at org.apache.hadoop.hdfs.DFSClient.setReplication(DFSClient.java:1443) at
org.apache.hadoop.hdfs.DistributedFileSystem$12.doCall(DistributedFileSystem.java:694)
at
org.apache.hadoop.hdfs.DistributedFileSystem$12.doCall(DistributedFileSystem.java:691)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.setReplication(DistributedFileSystem.java:701)
at
org.apache.hadoop.mapreduce.JobResourceUploader.copyJar(JobResourceUploader.java:787)
at
org.apache.hadoop.mapreduce.JobResourceUploader.uploadJobJar(JobResourceUploader.java:452)
at
org.apache.hadoop.mapreduce.JobResourceUploader.uploadResourcesInternal(JobResourceUploader.java:211)
at
org.apache.hadoop.mapreduce.JobResourceUploader.uploadResources(JobResourceUploader.java:135)
at
org.apache.hadoop.mapreduce.JobSubmitter.copyAndConfigureFiles(JobSubmitter.java:99)
at
org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:194)
at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1565) at
org.apache.hadoop.mapreduce.Job$11.run(Job.java:1562) at
java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1562) at
org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:576) at
org.apache.hadoop.mapred.JobClient$1.run(JobClient.java:571) at
java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.hadoop.mapred.JobClient.submitJobInternal(JobClient.java:571)
at org.apache.hadoop.mapred.JobClient.submitJob(JobClient.java:562) at
org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:873) at
com.skysider.tech.TestMapredTask.main(TestMapredTask.java:54) at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.apache.hadoop.util.RunJar.run(RunJar.java:323) at
org.apache.hadoop.util.RunJar.main(RunJar.java:236)Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.AccessControlException):
Permission denied: user=zds, access=WRITE,
inode="/tmp/zds/.staging/job_1645669537085_0002/job.jar":zds:supergroup:----rw-rw-
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:399)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:261)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:193)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1896)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1880)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPathAccess(FSDirectory.java:1830)
at
org.apache.hadoop.hdfs.server.namenode.FSDirAttrOp.setReplication(FSDirAttrOp.java:136)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.setReplication(FSNamesystem.java:2237)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.setReplication(NameNodeRpcServer.java:856)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.setReplication(ClientNamenodeProtocolServerSideTranslatorPB.java:524)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1086) at
org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1029) at
org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:957) at
java.security.AccessController.doPrivileged(Native Method) at
javax.security.auth.Subject.doAs(Subject.java:422) at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2957)
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1562)
at org.apache.hadoop.ipc.Client.call(Client.java:1508) at
org.apache.hadoop.ipc.Client.call(Client.java:1405) at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)
at com.sun.proxy.$Proxy9.setReplication(Unknown Source) at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.setReplication(ClientNamenodeProtocolTranslatorPB.java:422)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
at
org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
at com.sun.proxy.$Proxy10.setReplication(Unknown Source) at
org.apache.hadoop.hdfs.DFSClient.setReplication(DFSClient.java:1441) ... 31
more {code}
> MapReduce set permission too late in copyJar method
> ---------------------------------------------------
>
> Key: MAPREDUCE-7372
> URL: https://issues.apache.org/jira/browse/MAPREDUCE-7372
> Project: Hadoop Map/Reduce
> Issue Type: Bug
> Components: mrv2
> Affects Versions: 3.3.1
> Reporter: Zhang Dongsheng
> Priority: Major
> Attachments: 1.patch, MAPREDUCE-7372.patch
>
>
> while execute copyJar in JobResourceUploader .the setPermission running after
> setReplication,but setReplication need permission first.So if we set restrict
> umask in project such as 0600, the mapreduce process will fail.
> In patch file , I put setPermisson before setReplication.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
