ResumeCheckpointManuallyITCase failed with File X does not exist or the user running Flink C has insufficient permissions to access it

Caused by: java.io.FileNotFoundException: File file:/tmp/junit5368809541399217009/junit9107863722486384012/5a045e6c0cd0297faf5a2bf6fff27465/shared/job_5a045e6c0cd0297faf5a2bf6fff27465_op_90bea66de1c231edf33913ecd54406c1_1_2/0effb888-aa59-4bc4-b3e6-02622c831863 does not exist or the user running Flink ('agent01_azpcontainer') has insufficient permissions to access it.

2024-07-11T13:49:46.4137693Z Jul 11 13:49:46 13:49:46.412 [ERROR] Tests run: 48, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 309.7 s <<< FAILURE! -- in org.apache.flink.test.checkpointing.ResumeCheckpointManuallyITCase
2024-07-11T13:49:46.4139710Z Jul 11 13:49:46 13:49:46.412 [ERROR] org.apache.flink.test.checkpointing.ResumeCheckpointManuallyITCase.testExternalizedIncrementalRocksDBCheckpointsWithLocalRecoveryZookeeper[RestoreMode = CLAIM] -- Time elapsed: 2.722 s <<< ERROR!
2024-07-11T13:49:46.4140928Z Jul 11 13:49:46 org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy
2024-07-11T13:49:46.4142766Z Jul 11 13:49:46 	at org.apache.flink.runtime.executiongraph.failover.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:219)
2024-07-11T13:49:46.4144185Z Jul 11 13:49:46 	at org.apache.flink.runtime.executiongraph.failover.ExecutionFailureHandler.handleFailureAndReport(ExecutionFailureHandler.java:166)
2024-07-11T13:49:46.4145249Z Jul 11 13:49:46 	at org.apache.flink.runtime.executiongraph.failover.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:121)
2024-07-11T13:49:46.4146510Z Jul 11 13:49:46 	at org.apache.flink.runtime.scheduler.DefaultScheduler.recordTaskFailure(DefaultScheduler.java:281)
2024-07-11T13:49:46.4147599Z Jul 11 13:49:46 	at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:272)
2024-07-11T13:49:46.4148975Z Jul 11 13:49:46 	at org.apache.flink.runtime.scheduler.DefaultScheduler.onTaskFailed(DefaultScheduler.java:265)
2024-07-11T13:49:46.4150467Z Jul 11 13:49:46 	at org.apache.flink.runtime.scheduler.SchedulerBase.onTaskExecutionStateUpdate(SchedulerBase.java:800)
2024-07-11T13:49:46.4151977Z Jul 11 13:49:46 	at org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:777)
2024-07-11T13:49:46.4153308Z Jul 11 13:49:46 	at org.apache.flink.runtime.scheduler.SchedulerNG.updateTaskExecutionState(SchedulerNG.java:83)
2024-07-11T13:49:46.4154713Z Jul 11 13:49:46 	at org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:515)
2024-07-11T13:49:46.4155416Z Jul 11 13:49:46 	at sun.reflect.GeneratedMethodAccessor29.invoke(Unknown Source)
2024-07-11T13:49:46.4156342Z Jul 11 13:49:46 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2024-07-11T13:49:46.4157291Z Jul 11 13:49:46 	at java.lang.reflect.Method.invoke(Method.java:498)
2024-07-11T13:49:46.4158065Z Jul 11 13:49:46 	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.lambda$handleRpcInvocation$1(PekkoRpcActor.java:318)
2024-07-11T13:49:46.4159387Z Jul 11 13:49:46 	at org.apache.flink.runtime.concurrent.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:83)
2024-07-11T13:49:46.4160469Z Jul 11 13:49:46 	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.handleRpcInvocation(PekkoRpcActor.java:316)
2024-07-11T13:49:46.4161819Z Jul 11 13:49:46 	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.handleRpcMessage(PekkoRpcActor.java:229)
2024-07-11T13:49:46.4163253Z Jul 11 13:49:46 	at org.apache.flink.runtime.rpc.pekko.FencedPekkoRpcActor.handleRpcMessage(FencedPekkoRpcActor.java:88)
2024-07-11T13:49:46.4164717Z Jul 11 13:49:46 	at org.apache.flink.runtime.rpc.pekko.PekkoRpcActor.handleMessage(PekkoRpcActor.java:174)
2024-07-11T13:49:46.4165948Z Jul 11 13:49:46 	at org.apache.pekko.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:33)
2024-07-11T13:49:46.4167080Z Jul 11 13:49:46 	at org.apache.pekko.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:29)
2024-07-11T13:49:46.4168228Z Jul 11 13:49:46 	at scala.PartialFunction.applyOrElse(PartialFunction.scala:127)
2024-07-11T13:49:46.4169380Z Jul 11 13:49:46 	at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126)
2024-07-11T13:49:46.4170327Z Jul 11 13:49:46 	at org.apache.pekko.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:29)
2024-07-11T13:49:46.4171192Z Jul 11 13:49:46 	at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:175)
2024-07-11T13:49:46.4171814Z Jul 11 13:49:46 	at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176)
2024-07-11T13:49:46.4172433Z Jul 11 13:49:46 	at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176)
2024-07-11T13:49:46.4173029Z Jul 11 13:49:46 	at org.apache.pekko.actor.Actor.aroundReceive(Actor.scala:547)
2024-07-11T13:49:46.4173622Z Jul 11 13:49:46 	at org.apache.pekko.actor.Actor.aroundReceive$(Actor.scala:545)
2024-07-11T13:49:46.4174236Z Jul 11 13:49:46 	at org.apache.pekko.actor.AbstractActor.aroundReceive(AbstractActor.scala:229)
2024-07-11T13:49:46.4174948Z Jul 11 13:49:46 	at org.apache.pekko.actor.ActorCell.receiveMessage(ActorCell.scala:590)
2024-07-11T13:49:46.4175546Z Jul 11 13:49:46 	at org.apache.pekko.actor.ActorCell.invoke(ActorCell.scala:557)
2024-07-11T13:49:46.4176145Z Jul 11 13:49:46 	at org.apache.pekko.dispatch.Mailbox.processMailbox(Mailbox.scala:280)
2024-07-11T13:49:46.4176713Z Jul 11 13:49:46 	at org.apache.pekko.dispatch.Mailbox.run(Mailbox.scala:241)
2024-07-11T13:49:46.4177276Z Jul 11 13:49:46 	at org.apache.pekko.dispatch.Mailbox.exec(Mailbox.scala:253)
2024-07-11T13:49:46.4177860Z Jul 11 13:49:46 	at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
2024-07-11T13:49:46.4178642Z Jul 11 13:49:46 	at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
2024-07-11T13:49:46.4179281Z Jul 11 13:49:46 	at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
2024-07-11T13:49:46.4179984Z Jul 11 13:49:46 	at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
2024-07-11T13:49:46.4182436Z Jul 11 13:49:46 Caused by: java.lang.RuntimeException: java.io.FileNotFoundException: File file:/tmp/junit5368809541399217009/junit9107863722486384012/5a045e6c0cd0297faf5a2bf6fff27465/shared/job_5a045e6c0cd0297faf5a2bf6fff27465_op_90bea66de1c231edf33913ecd54406c1_1_2/0effb888-aa59-4bc4-b3e6-02622c831863 does not exist or the user running Flink ('agent01_azpcontainer') has insufficient permissions to access it.
2024-07-11T13:49:46.4183912Z Jul 11 13:49:46 	at org.apache.flink.runtime.checkpoint.filemerging.FileMergingSnapshotManagerBase.lambda$null$11(FileMergingSnapshotManagerBase.java:880)
2024-07-11T13:49:46.4184756Z Jul 11 13:49:46 	at java.util.HashMap.computeIfAbsent(HashMap.java:1127)
2024-07-11T13:49:46.4185539Z Jul 11 13:49:46 	at org.apache.flink.runtime.checkpoint.filemerging.FileMergingSnapshotManagerBase.lambda$restoreStateHandles$12(FileMergingSnapshotManagerBase.java:861)
2024-07-11T13:49:46.4186506Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2024-07-11T13:49:46.4187334Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2024-07-11T13:49:46.4187985Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
2024-07-11T13:49:46.4188626Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2024-07-11T13:49:46.4189243Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2024-07-11T13:49:46.4189875Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
2024-07-11T13:49:46.4190674Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2024-07-11T13:49:46.4191362Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2024-07-11T13:49:46.4192025Z Jul 11 13:49:46 	at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
2024-07-11T13:49:46.4192674Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2024-07-11T13:49:46.4193294Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2024-07-11T13:49:46.4193948Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2024-07-11T13:49:46.4194707Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2024-07-11T13:49:46.4195361Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2024-07-11T13:49:46.4195980Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2024-07-11T13:49:46.4196608Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2024-07-11T13:49:46.4197264Z Jul 11 13:49:46 	at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)
2024-07-11T13:49:46.4197897Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2024-07-11T13:49:46.4198550Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2024-07-11T13:49:46.4199497Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2024-07-11T13:49:46.4200461Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2024-07-11T13:49:46.4201487Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2024-07-11T13:49:46.4202420Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2024-07-11T13:49:46.4203455Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2024-07-11T13:49:46.4204673Z Jul 11 13:49:46 	at java.util.ArrayList$Itr.forEachRemaining(ArrayList.java:901)
2024-07-11T13:49:46.4205473Z Jul 11 13:49:46 	at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
2024-07-11T13:49:46.4206162Z Jul 11 13:49:46 	at java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:742)
2024-07-11T13:49:46.4206786Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2024-07-11T13:49:46.4207425Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2024-07-11T13:49:46.4208069Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2024-07-11T13:49:46.4208724Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2024-07-11T13:49:46.4209387Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2024-07-11T13:49:46.4210232Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2024-07-11T13:49:46.4211005Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2024-07-11T13:49:46.4211665Z Jul 11 13:49:46 	at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)
2024-07-11T13:49:46.4212301Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2024-07-11T13:49:46.4212918Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2024-07-11T13:49:46.4213568Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2024-07-11T13:49:46.4214236Z Jul 11 13:49:46 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2024-07-11T13:49:46.4214959Z Jul 11 13:49:46 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2024-07-11T13:49:46.4215578Z Jul 11 13:49:46 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2024-07-11T13:49:46.4216366Z Jul 11 13:49:46 	at org.apache.flink.runtime.checkpoint.filemerging.FileMergingSnapshotManagerBase.restoreStateHandles(FileMergingSnapshotManagerBase.java:858)
2024-07-11T13:49:46.4217323Z Jul 11 13:49:46 	at org.apache.flink.runtime.checkpoint.filemerging.SubtaskFileMergingManagerRestoreOperation.restore(SubtaskFileMergingManagerRestoreOperation.java:102)
2024-07-11T13:49:46.4218329Z Jul 11 13:49:46 	at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.registerRestoredStateToFileMergingManager(StreamTaskStateInitializerImpl.java:353)
2024-07-11T13:49:46.4219299Z Jul 11 13:49:46 	at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:163)
2024-07-11T13:49:46.4220233Z Jul 11 13:49:46 	at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:275)
2024-07-11T13:49:46.4221220Z Jul 11 13:49:46 	at org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.initializeStateAndOpenOperators(RegularOperatorChain.java:106)
2024-07-11T13:49:46.4222025Z Jul 11 13:49:46 	at org.apache.flink.streaming.runtime.tasks.StreamTask.restoreStateAndGates(StreamTask.java:858)
2024-07-11T13:49:46.4222769Z Jul 11 13:49:46 	at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$restoreInternal$5(StreamTask.java:812)
2024-07-11T13:49:46.4223540Z Jul 11 13:49:46 	at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.call(StreamTaskActionExecutor.java:55)
2024-07-11T13:49:46.4224277Z Jul 11 13:49:46 	at org.apache.flink.streaming.runtime.tasks.StreamTask.restoreInternal(StreamTask.java:812)
2024-07-11T13:49:46.4225039Z Jul 11 13:49:46 	at org.apache.flink.streaming.runtime.tasks.StreamTask.restore(StreamTask.java:771)
2024-07-11T13:49:46.4225822Z Jul 11 13:49:46 	at org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:970)
2024-07-11T13:49:46.4226470Z Jul 11 13:49:46 	at org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:939)
2024-07-11T13:49:46.4227085Z Jul 11 13:49:46 	at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:763)
2024-07-11T13:49:46.4227668Z Jul 11 13:49:46 	at org.apache.flink.runtime.taskmanager.Task.run(Task.java:575)
2024-07-11T13:49:46.4228180Z Jul 11 13:49:46 	at java.lang.Thread.run(Thread.java:748)
2024-07-11T13:49:46.4229980Z Jul 11 13:49:46 Caused by: java.io.FileNotFoundException: File file:/tmp/junit5368809541399217009/junit9107863722486384012/5a045e6c0cd0297faf5a2bf6fff27465/shared/job_5a045e6c0cd0297faf5a2bf6fff27465_op_90bea66de1c231edf33913ecd54406c1_1_2/0effb888-aa59-4bc4-b3e6-02622c831863 does not exist or the user running Flink ('agent01_azpcontainer') has insufficient permissions to access it.
2024-07-11T13:49:46.4231378Z Jul 11 13:49:46 	at org.apache.flink.core.fs.local.LocalFileSystem.getFileStatus(LocalFileSystem.java:106)
2024-07-11T13:49:46.4232119Z Jul 11 13:49:46 	at org.apache.flink.core.fs.SafetyNetWrapperFileSystem.getFileStatus(SafetyNetWrapperFileSystem.java:78)
2024-07-11T13:49:46.4233041Z Jul 11 13:49:46 	at org.apache.flink.runtime.checkpoint.filemerging.FileMergingSnapshotManagerBase.getFileSize(FileMergingSnapshotManagerBase.java:910)
2024-07-11T13:49:46.4233973Z Jul 11 13:49:46 	at org.apache.flink.runtime.checkpoint.filemerging.FileMergingSnapshotManagerBase.lambda$null$11(FileMergingSnapshotManagerBase.java:878)
2024-07-11T13:49:46.4234689Z Jul 11 13:49:46 	... 59 more