flinkx run with flink on yarn perjob mode, the command and job config as follows. it does not throw an exception, and blocked in YarnClusterDescriptor.deployJobCluster. what is the reason of this probolem.
bin/flinkx -mode yarnPer -job ./job/demo.json -pluginRoot /home/yifanzhang/flinkx/plugins -flinkconf /home/yifanzhang/flink-1.8.1/conf -yarnconf /opt/cloudera/parcels/CDH/lib/hadoop-yarn/etc/hadoop -flinkLibJar /home/yifanzhang/flink/lib
{
"job" : {
"content" : [ {
"reader" : {
"parameter" : {
"groupId" : "default",
"topic" : "test",
"consumerSettings" : {
"zookeeper.connect" : "10.126.144.37:2181,10.126.144.38:2181,10.126.144.39:2181/cp3",
"bootstrap.servers" : "10.126.144.34:9092,10.126.144.35:9092,10.126.144.36:9092",
"auto.commit.interval.ms" : "1000",
"auto.offset.reset" : "latest"
}
},
"name" : "kafka11reader",
"type" : 14
},
"writer" : {
"parameter" : {
"path" : "/tmp",
"fileName" : "test",
"hadoopConfig" : {
"dfs.ha.namenodes.ns1" : "namenode252,namenode369",
"fs.defaultFS" : "hdfs:///idc-nn",
"dfs.namenode.rpc-address.idc-nn.namenode252" : "shoyi04plpebdp015.mcd.com.cn:8020",
"dfs.client.failover.proxy.provider.idc-nn" : "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"dfs.namenode.rpc-address.idc-nn.namenode369" : "shoyi05plpebdp019.mcd.com.cn:8022",
"dfs.nameservices" : "idc-nn",
"fs.hdfs.impl.disable.cache" : "true",
"fs.hdfs.impl" : "org.apache.hadoop.hdfs.DistributedFileSystem",
"hadoop.security.authorization": "true",
"hadoop.security.authentication": "Kerberos",
"dfs.namenode.kerberos.principal": "yifanzhang",
"dfs.namenode.keytab.file": "/home/yifanzhang/yifanzhang.keytab",
"java.security.krb5.conf": "/etc/krb5.conf"
},
"column" : [ {
"name" : "id",
"index" : 0,
"type" : "string"
} ],
"defaultFS" : "hdfs:///idc-nn",
"writeMode" : "APPEND",
"fieldDelimiter" : ",",
"fileType" : "orc",
"charsetName" : "utf-8"
},
"name" : "hdfswriter",
"type" : 6
}
} ],
"setting" : {
"restore" : {
"isRestore" : true,
"isStream" : true
},
"errorLimit" : {
},
"speed" : {
"bytes" : 2097152,
"channel" : 3
}
}
}
}
flinkx run with flink on yarn perjob mode, the command and job config as follows. it does not throw an exception, and blocked in YarnClusterDescriptor.deployJobCluster. what is the reason of this probolem.
bin/flinkx -mode yarnPer -job ./job/demo.json -pluginRoot /home/yifanzhang/flinkx/plugins -flinkconf /home/yifanzhang/flink-1.8.1/conf -yarnconf /opt/cloudera/parcels/CDH/lib/hadoop-yarn/etc/hadoop -flinkLibJar /home/yifanzhang/flink/lib
{
"job" : {
"content" : [ {
"reader" : {
"parameter" : {
"groupId" : "default",
"topic" : "test",
"consumerSettings" : {
"zookeeper.connect" : "10.126.144.37:2181,10.126.144.38:2181,10.126.144.39:2181/cp3",
"bootstrap.servers" : "10.126.144.34:9092,10.126.144.35:9092,10.126.144.36:9092",
"auto.commit.interval.ms" : "1000",
"auto.offset.reset" : "latest"
}
},
"name" : "kafka11reader",
"type" : 14
},
"writer" : {
"parameter" : {
"path" : "/tmp",
"fileName" : "test",
"hadoopConfig" : {
"dfs.ha.namenodes.ns1" : "namenode252,namenode369",
"fs.defaultFS" : "hdfs:///idc-nn",
"dfs.namenode.rpc-address.idc-nn.namenode252" : "shoyi04plpebdp015.mcd.com.cn:8020",
"dfs.client.failover.proxy.provider.idc-nn" : "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"dfs.namenode.rpc-address.idc-nn.namenode369" : "shoyi05plpebdp019.mcd.com.cn:8022",
"dfs.nameservices" : "idc-nn",
"fs.hdfs.impl.disable.cache" : "true",
"fs.hdfs.impl" : "org.apache.hadoop.hdfs.DistributedFileSystem",
"hadoop.security.authorization": "true",
"hadoop.security.authentication": "Kerberos",
"dfs.namenode.kerberos.principal": "yifanzhang",
"dfs.namenode.keytab.file": "/home/yifanzhang/yifanzhang.keytab",
"java.security.krb5.conf": "/etc/krb5.conf"
},
"column" : [ {
"name" : "id",
"index" : 0,
"type" : "string"
} ],
"defaultFS" : "hdfs:///idc-nn",
"writeMode" : "APPEND",
"fieldDelimiter" : ",",
"fileType" : "orc",
"charsetName" : "utf-8"
},
"name" : "hdfswriter",
"type" : 6
}
} ],
"setting" : {
"restore" : {
"isRestore" : true,
"isStream" : true
},
"errorLimit" : {
},
"speed" : {
"bytes" : 2097152,
"channel" : 3
}
}
}
}