28732
DISCUSSIONS
101748
MEMBERS
3157
ARTICLES
Running CDH 5.7.3 with Kerberos, TLS/SSL level 1, and TDE/Key Trustee KMS. Have a Key Trustee Server Cluster. Everything works fine. The kms ticket liefetime is set to 7 days
hadoop.kms.authentication.delegation-token.max-lifetime.sec
After 7 days the token expires, preventing any further work. The application is a long running process where the user has loggged out. What is the best practice for renewing the ticket?
Thanks
The stack after 7 days:
016-11-28 19:42:51,048 ERROR AttivioEngine [EngineServerThread-12962] - ATTIVIO-INDEX_ENGINE-41 : [index.writer-part2-ba72f394-abed-4c8d-aefd-3212c96a5b6d] Fatal error occurred while indexing org.apache.hadoop.security.authentication.client.AuthenticationException - org.apache.hadoop.security.token.SecretManager$InvalidToken: token (kms-dt owner=systemtest, renewer=yarn, realUser=, issueDate=1479767372233, maxDate=1480372172233, sequenceNumber=320, masterKeyId=13) is expired org.apache.hadoop.security.authentication.client.AuthenticationException: org.apache.hadoop.security.token.SecretManager$InvalidToken: token (kms-dt owner=systemtest, renewer=yarn, realUser=, issueDate=1479767372233, maxDate=1480372172233, sequenceNumber=320, masterKeyId=13) is expired at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at org.apache.hadoop.util.HttpExceptionUtils.validateResponse(HttpExceptionUtils.java:157) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:546) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:504) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(KMSClientProvider.java:779) at org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncryptedKey(KeyProviderCryptoExtension.java:388) at org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(DFSClient.java:1381) at org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1483) at org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1468) at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:451) at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:444) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:459) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:956) at com.attivio.lucene.store.hadoop.HadoopDirectory.createOutput(HadoopDirectory.java:90) at org.apache.lucene.store.NRTCachingDirectory.createOutput(NRTCachingDirectory.java:156) at com.attivio.lucene.store.AttivioDirectory.createOutput(AttivioDirectory.java:231) at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) at org.apache.lucene.codecs.lucene50.Lucene50NormsConsumer.<init>(Lucene50NormsConsumer.java:64) at org.apache.lucene.codecs.lucene50.Lucene50NormsFormat.normsConsumer(Lucene50NormsFormat.java:123) at org.apache.lucene.index.DefaultIndexingChain.writeNorms(DefaultIndexingChain.java:196) at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:95) at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:420) at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:512) at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:624) at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2702) at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:2866) at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:2833) at org.apache.lucene.index.AttivioIndexWriter.commit(AttivioIndexWriter.java:67) at com.attivio.lucene.index.Indexer.doCommit(Indexer.java:346) at com.attivio.lucene.index.DocumentIndexer.commit(DocumentIndexer.java:209) at com.attivio.lucene.index.RealTimeZone.commit(RealTimeZone.java:396) at com.attivio.lucene.index.ft.FaultTolerantZone.commit(FaultTolerantZone.java:288) at com.attivio.lucene.index.IndexCore.commit(IndexCore.java:729) at com.attivio.platform.engine.AttivioEngine.startCommit(AttivioEngine.java:1444) at com.attivio.platform.engine.AttivioEngine.access$1000(AttivioEngine.java:90) at com.attivio.platform.engine.AttivioEngine$IndexingSession.commit(AttivioEngine.java:1353) at com.attivio.platform.engine.AttivioEngine$IndexingSession.process(AttivioEngine.java:1121) at com.attivio.platform.engine.ContentRequestHandler$MessageProcessor.call(ContentRequestHandler.java:434) at com.attivio.platform.engine.ContentRequestHandler$DispatcherInputStream.receiveMessage(ContentRequestHandler.java:366) at com.attivio.platform.engine.ContentRequestHandler.handle(ContentRequestHandler.java:73) at com.attivio.platform.engine.EngineServer$Dispatcher.run(EngineServer.java:533) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at com.attivio.platform.engine.EngineServer$ThreadFactoryRunnable.run(EngineServer.java:603) at java.lang.Thread.run(Thread.java:745) 2016-11-28 19:42:51,499 WARN ContentRequestHandler [EngineServerThread-12962] - ATTIVIO-INDEX_ENGINE-23 : [/index] Node cae77489-3dd0-4e03-b739-be440bb6b17c: Engine writer-part2-ba72f394-abed-4c8d-aefd-3212c96a5b6d offline 2016-11-28 19:42:51,500 ERROR AieIndexLauncher [Thread-603372] - ATTIVIO-PLATFORM-24 : Uncaught thread death java.lang.ThreadGroup[name=EngineServer,maxpri=10]:Thread-603372 org.apache.hadoop.security.authentication.client.AuthenticationException - org.apache.hadoop.security.token.SecretManager$InvalidToken: token (kms-dt owner=systemtest, renewer=yarn, realUser=, issueDate=1479767372233, maxDate=1480372172233, sequenceNumber=320, masterKeyId=13) can't be found in cache org.apache.hadoop.security.authentication.client.AuthenticationException: org.apache.hadoop.security.token.SecretManager$InvalidToken: token (kms-dt owner=systemtest, renewer=yarn, realUser=, issueDate=1479767372233, maxDate=1480372172233, sequenceNumber=320, masterKeyId=13) can't be found in cache at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:422) at org.apache.hadoop.util.HttpExceptionUtils.validateResponse(HttpExceptionUtils.java:157) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:546) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.call(KMSClientProvider.java:504) at org.apache.hadoop.crypto.key.kms.KMSClientProvider.decryptEncryptedKey(KMSClientProvider.java:779) at org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.decryptEncryptedKey(KeyProviderCryptoExtension.java:388) at org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(DFSClient.java:1381) at org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1483) at org.apache.hadoop.hdfs.DFSClient.createWrappedOutputStream(DFSClient.java:1468) at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:451) at org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:444) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:459) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:956) at com.attivio.lucene.store.hadoop.HadoopDirectory.createOutput(HadoopDirectory.java:90) at org.apache.lucene.store.NRTCachingDirectory.unCache(NRTCachingDirectory.java:249) at org.apache.lucene.store.NRTCachingDirectory.close(NRTCachingDirectory.java:207) at com.attivio.lucene.store.AttivioDirectory.close(AttivioDirectory.java:263) at com.attivio.lucene.index.DocumentIndexer.shutdown(DocumentIndexer.java:233) at com.attivio.lucene.index.RealTimeZone.shutdown(RealTimeZone.java:470) at com.attivio.lucene.index.ft.FaultTolerantZone.shutdown(FaultTolerantZone.java:339) at com.attivio.lucene.index.IndexCore.shutdown(IndexCore.java:847) at com.attivio.platform.engine.AttivioEngine.stopComponentInternal(AttivioEngine.java:810) at com.attivio.platform.engine.AttivioEngine.stopComponent(AttivioEngine.java:779) at com.attivio.platform.engine.AttivioEngine$ShutdownThread.run(AttivioEngine.java:745)