Synchronizing local compiler cache.

neuronxcc-2.16.345.0+69131dd3/0_REGISTRY/0.0.28.dev0/training/llama/michaelbenayoun/llama-2-tiny-4kv-heads-4layers-random/e4b7f2d4b9fdeed1fc72.json

@@ -0,0 +1 @@
+{"_attn_implementation_autoset": true, "architectures": ["LlamaForCausalLM"], "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "head_dim": 4, "hidden_act": "silu", "hidden_size": 128, "initializer_range": 0.02, "intermediate_size": 128, "is_decoder": true, "max_position_embeddings": 4096, "mlp_bias": false, "model_type": "llama", "neuron": {"compiler_version": "2.16.345.0+69131dd3", "input_specs": {"attention_mask": [1, 128], "input_ids": [1, 128], "labels": [1, 128]}, "model_class": "LlamaForCausalLM", "num_neuron_cores_per_node": 2, "pipeline_parallel_size": 1, "precision": "float32", "tensor_parallel_size": 1, "training": true}, "num_attention_heads": 32, "num_hidden_layers": 4, "num_key_value_heads": 4, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000.0, "tie_word_embeddings": false, "torch_dtype": "float32", "use_cache": true, "vocab_size": 32000}

neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_10351170258984126849+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:741e0b7e1cd35d27b3848f3974a0e2d479de880e8965741c0b9d8c0f02b5e3b5
+size 1848

neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_11264542452468788917+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4b81ff56bfc39354a840db1955192322509af7d679c975d8184d15a6e0fe5e0
+size 299

neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1"]

neuronxcc-2.16.345.0+69131dd3/MODULE_12281391252418677086+e30acd3a/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f79c2d87a0ddcf0b26820282a1a90e805a69b92404101a0963689d7e4308927b
+size 121417

neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1bc054c5f5ede2a2051ce74e48f607791b85eba3a90d541637b29079cbc87d2
+size 148332

neuronxcc-2.16.345.0+69131dd3/MODULE_12445205521823827895+3cc9a3cb/model.log

@@ -0,0 +1,57 @@
+Failed compilation with ['neuronx-cc', 'compile', '--framework=XLA', '/tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/model.MODULE_12445205521823827895+3cc9a3cb.hlo_module.pb', '--output', '/tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/model.MODULE_12445205521823827895+3cc9a3cb.neff', '--target=trn1', '--model-type=transformer', '--verbose=35']: Process Process-1:
+Traceback (most recent call last):
+  File "neuronxcc/driver/CommandDriver.py", line 345, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1353, in neuronxcc.driver.commands.CompileCommand.CompileCommand.run
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1304, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1324, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1327, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run
+  File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState
+  File "neuronxcc/driver/Pipeline.py", line 30, in neuronxcc.driver.Pipeline.Pipeline.runSingleInput
+  File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run
+  File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState
+  File "neuronxcc/driver/jobs/Frontend.py", line 454, in neuronxcc.driver.jobs.Frontend.Frontend.runSingleInput
+  File "neuronxcc/driver/jobs/Frontend.py", line 218, in neuronxcc.driver.jobs.Frontend.Frontend.runXLAFrontend
+  File "neuronxcc/driver/jobs/Frontend.py", line 190, in neuronxcc.driver.jobs.Frontend.Frontend.runHlo2Tensorizer
+neuronxcc.driver.Exceptions.CompilerInvalidInputException: ERROR: Failed command  /home/runner/_work/optimum-neuron/optimum-neuron/aws_neuron_venv_pytorch/lib/python3.10/site-packages/neuronxcc/starfish/bin/hlo2penguin --input /tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/model.MODULE_12445205521823827895+3cc9a3cb.hlo_module.pb --out-dir ./ --output penguin.py --remat --max-costly-ops=2 --max-live-in-size=5 --max-remat-chain-size=10 --max-mem-multiple=1.8 --min-def-use-distance=500 --remat-policy=transformer --allow-same-pass-remat=true --split-abc --layers-per-module=1 --emit-tensor-level-dropout-ops --emit-tensor-level-rng-ops --native-to-custom-softmax --partitioner-opts='--transformer'
+------------ 
+Reported stdout: 
+DEBUG: needsModular? No. macCnt 0
+INFO: Switching to single-module compile. PrePartitionPipe skipped.
+INFO: Found memory bound graph
+INFO: Number of Native SoftmaxDx's detected and replaced: 0
+INFO: Number of Native Softmax's detected and replaced: 0
+Replaced 0 dropout sequences with OffloadedDropout
+INFO: HloMacCount has found 0
+INFO: Traffic has found 68311702
+INFO: AIF 0
+HLO Ops used in computation: add all-reduce broadcast compare constant copy divide get-tuple-element multiply parameter power reduce reshape select sqrt subtract tuple 
+2025-01-07 14:52:23.605725: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %tuple.1 = tuple(%add.806, %copy.119, %add.902, %add.950, %copy.122, %add.1046, %copy.124, %copy.125, %copy.126, %copy.127, %copy.128, %copy.129, %copy.130, %copy.131, %copy.132, %copy.133, %copy.134, %copy.135, %copy.136, %copy.137, %copy.138, %copy.139, %copy.140, %copy.141, %copy.142, %copy.143, %copy.144, %copy.145, %copy.146, %copy.147, %add.2245, %copy.149, %copy.150, %copy.151, %copy.152, %copy.153, %copy.154, %copy.155, %copy.156, %constant.122, %copy.158, %copy.159, %copy.160, %copy.161, %copy.1... to 512 characters in the compiler's debug metadata
+Invoking RemoveOptimizationBarriers pass
+2025-01-07 14:52:23.618811: F hilo/MLIRPasses/Transforms/VerifyAliasing.cc:125] false && "Read after write on aliased input/output buffer."
+
+------------ 
+Reported stderr: 
+None
+------------                       
+Import of the HLO graph into the Neuron Compiler has failed.                       
+This may be caused by unsupported operators or an internal compiler error.                       
+More details can be found in the error message(s) above.
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "neuronxcc/driver/CommandDriver.py", line 352, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand_in_process
+  File "neuronxcc/driver/CommandDriver.py", line 347, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand
+  File "neuronxcc/driver/CommandDriver.py", line 111, in neuronxcc.driver.CommandDriver.handleError
+  File "neuronxcc/driver/GlobalState.py", line 102, in neuronxcc.driver.GlobalState.FinalizeGlobalState
+  File "neuronxcc/driver/GlobalState.py", line 82, in neuronxcc.driver.GlobalState._GlobalStateImpl.shutdown
+  File "/usr/lib/python3.10/shutil.py", line 715, in rmtree
+    onerror(os.lstat, path, sys.exc_info())
+  File "/usr/lib/python3.10/shutil.py", line 713, in rmtree
+    orig_st = os.lstat(path)
+FileNotFoundError: [Errno 2] No such file or directory: '/tmp/no-user/neuroncc_compile_workdir/c010323a-c1cc-4661-9aa1-91df76d78037/neuronxcc-9f3m6nhk'

neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_14152517080381013961+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a67865aabfb27a273f32eb2a882f6269447d64eb1e7a72583db03ddd45d87c1
+size 301

neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:113f044831a4e1c32f50de67f9bac777f9c1f5ce03532c84565e3c20b6d0c473
+size 141306

neuronxcc-2.16.345.0+69131dd3/MODULE_14466467378454838209+3cc9a3cb/model.log

@@ -0,0 +1,59 @@
+Failed compilation with ['neuronx-cc', 'compile', '--framework=XLA', '/tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/model.MODULE_14466467378454838209+3cc9a3cb.hlo_module.pb', '--output', '/tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/model.MODULE_14466467378454838209+3cc9a3cb.neff', '--target=trn1', '--model-type=transformer', '--verbose=35']: Process Process-1:
+Traceback (most recent call last):
+  File "neuronxcc/driver/CommandDriver.py", line 345, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1353, in neuronxcc.driver.commands.CompileCommand.CompileCommand.run
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1304, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1324, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1327, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run
+  File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState
+  File "neuronxcc/driver/Pipeline.py", line 30, in neuronxcc.driver.Pipeline.Pipeline.runSingleInput
+  File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run
+  File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState
+  File "neuronxcc/driver/jobs/Frontend.py", line 454, in neuronxcc.driver.jobs.Frontend.Frontend.runSingleInput
+  File "neuronxcc/driver/jobs/Frontend.py", line 218, in neuronxcc.driver.jobs.Frontend.Frontend.runXLAFrontend
+  File "neuronxcc/driver/jobs/Frontend.py", line 190, in neuronxcc.driver.jobs.Frontend.Frontend.runHlo2Tensorizer
+neuronxcc.driver.Exceptions.CompilerInvalidInputException: ERROR: Failed command  /home/runner/_work/optimum-neuron/optimum-neuron/aws_neuron_venv_pytorch/lib/python3.10/site-packages/neuronxcc/starfish/bin/hlo2penguin --input /tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/model.MODULE_14466467378454838209+3cc9a3cb.hlo_module.pb --out-dir ./ --output penguin.py --remat --max-costly-ops=2 --max-live-in-size=5 --max-remat-chain-size=10 --max-mem-multiple=1.8 --min-def-use-distance=500 --remat-policy=transformer --allow-same-pass-remat=true --split-abc --layers-per-module=1 --emit-tensor-level-dropout-ops --emit-tensor-level-rng-ops --native-to-custom-softmax --partitioner-opts='--transformer'
+------------ 
+Reported stdout: 
+DEBUG: needsModular? No. macCnt 0
+INFO: Switching to single-module compile. PrePartitionPipe skipped.
+INFO: Found memory bound graph
+INFO: Number of Native SoftmaxDx's detected and replaced: 0
+INFO: Number of Native Softmax's detected and replaced: 0
+Replaced 0 dropout sequences with OffloadedDropout
+INFO: HloMacCount has found 0
+INFO: Traffic has found 170744446
+INFO: AIF 0
+HLO Ops used in computation: add all-reduce broadcast compare concatenate constant copy divide get-tuple-element multiply parameter reduce reshape select sqrt subtract tuple 
+2025-01-07 14:48:59.478813: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %concatenate.439 = concatenate(%reshape.400, %reshape.401, %reshape.402, %reshape.403, %reshape.404, %reshape.405, %reshape.406, %reshape.407, %reshape.408, %reshape.409, %reshape.410, %reshape.411, %reshape.412, %reshape.413, %reshape.414, %reshape.415, %reshape.416, %reshape.417, %reshape.418, %reshape.419, %reshape.420, %reshape.421, %reshape.422, %reshape.423, %reshape.424, %reshape.425, %reshape.426, %reshape.427, %reshape.428, %reshape.429, %reshape.430, %reshape.431, %reshape.432, %reshape.433, %r... to 512 characters in the compiler's debug metadata
+2025-01-07 14:48:59.479598: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %all-reduce.580 = all-reduce(%multiply.534, %multiply.532, %multiply.530, %multiply.528, %multiply.526, %multiply.524, %multiply.522, %multiply.520, %multiply.518, %multiply.516, %multiply.514, %multiply.512, %multiply.510, %multiply.508, %multiply.506, %multiply.504, %multiply.502, %multiply.500, %multiply.498, %multiply.496, %multiply.494, %multiply.492, %multiply.490, %multiply.488, %multiply.486, %multiply.484, %multiply.482, %multiply.480, %multiply.478, %multiply.476, %multiply.474, %multiply.472, ... to 512 characters in the compiler's debug metadata
+2025-01-07 14:48:59.479761: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %tuple.1 = tuple(%add.774, %add.808, %add.842, %copy.121, %copy.122, %copy.123, %copy.124, %copy.125, %add.1046, %copy.127, %copy.128, %copy.129, %copy.130, %copy.131, %copy.132, %copy.133, %copy.134, %copy.135, %copy.136, %copy.137, %copy.138, %copy.139, %copy.140, %copy.141, %copy.142, %copy.143, %copy.144, %copy.145, %copy.146, %copy.147, %copy.148, %copy.149, %copy.150, %copy.151, %copy.152, %copy.153, %copy.154, %copy.155, %copy.156, %constant.12, %copy.158, %copy.159, %copy.160, %copy.161, %copy.16... to 512 characters in the compiler's debug metadata
+Invoking RemoveOptimizationBarriers pass
+2025-01-07 14:48:59.492367: F hilo/MLIRPasses/Transforms/VerifyAliasing.cc:125] false && "Read after write on aliased input/output buffer."
+
+------------ 
+Reported stderr: 
+None
+------------                       
+Import of the HLO graph into the Neuron Compiler has failed.                       
+This may be caused by unsupported operators or an internal compiler error.                       
+More details can be found in the error message(s) above.
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "neuronxcc/driver/CommandDriver.py", line 352, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand_in_process
+  File "neuronxcc/driver/CommandDriver.py", line 347, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand
+  File "neuronxcc/driver/CommandDriver.py", line 111, in neuronxcc.driver.CommandDriver.handleError
+  File "neuronxcc/driver/GlobalState.py", line 102, in neuronxcc.driver.GlobalState.FinalizeGlobalState
+  File "neuronxcc/driver/GlobalState.py", line 82, in neuronxcc.driver.GlobalState._GlobalStateImpl.shutdown
+  File "/usr/lib/python3.10/shutil.py", line 715, in rmtree
+    onerror(os.lstat, path, sys.exc_info())
+  File "/usr/lib/python3.10/shutil.py", line 713, in rmtree
+    orig_st = os.lstat(path)
+FileNotFoundError: [Errno 2] No such file or directory: '/tmp/no-user/neuroncc_compile_workdir/9a13359e-4fac-4cca-91ac-310afbe0c188/neuronxcc-i3pdyt1j'

neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_14568724634861814049+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b83fbb315057e53957c7f172e7d7a5deeda3955f7c8625082ad4b2c0a55a5a6a
+size 118841

neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1"]

neuronxcc-2.16.345.0+69131dd3/MODULE_14596302589284321818+e30acd3a/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0afb122b426ab077b2d37431b30bc1bff549e04be1b825219b2eeb0973ffa46
+size 1792

neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_15105893443766915069+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4385afb5b7af6963fa2374ea47ac8eb2ec073c8242b564f970bc65cd8a13eab1
+size 126812

neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58bcd7af24a3d214372af69d0fbefa4e192482762d9e75b0dab4cdb7a5144964
+size 151549

neuronxcc-2.16.345.0+69131dd3/MODULE_1590662263737562288+3cc9a3cb/model.log

@@ -0,0 +1,59 @@
+Failed compilation with ['neuronx-cc', 'compile', '--framework=XLA', '/tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/model.MODULE_1590662263737562288+3cc9a3cb.hlo_module.pb', '--output', '/tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/model.MODULE_1590662263737562288+3cc9a3cb.neff', '--target=trn1', '--model-type=transformer', '--verbose=35']: Process Process-1:
+Traceback (most recent call last):
+  File "neuronxcc/driver/CommandDriver.py", line 345, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1353, in neuronxcc.driver.commands.CompileCommand.CompileCommand.run
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1304, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1324, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/commands/CompileCommand.py", line 1327, in neuronxcc.driver.commands.CompileCommand.CompileCommand.runPipeline
+  File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run
+  File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState
+  File "neuronxcc/driver/Pipeline.py", line 30, in neuronxcc.driver.Pipeline.Pipeline.runSingleInput
+  File "neuronxcc/driver/Job.py", line 344, in neuronxcc.driver.Job.SingleInputJob.run
+  File "neuronxcc/driver/Job.py", line 370, in neuronxcc.driver.Job.SingleInputJob.runOnState
+  File "neuronxcc/driver/jobs/Frontend.py", line 454, in neuronxcc.driver.jobs.Frontend.Frontend.runSingleInput
+  File "neuronxcc/driver/jobs/Frontend.py", line 218, in neuronxcc.driver.jobs.Frontend.Frontend.runXLAFrontend
+  File "neuronxcc/driver/jobs/Frontend.py", line 190, in neuronxcc.driver.jobs.Frontend.Frontend.runHlo2Tensorizer
+neuronxcc.driver.Exceptions.CompilerInvalidInputException: ERROR: Failed command  /home/runner/_work/optimum-neuron/optimum-neuron/aws_neuron_venv_pytorch/lib/python3.10/site-packages/neuronxcc/starfish/bin/hlo2penguin --input /tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/model.MODULE_1590662263737562288+3cc9a3cb.hlo_module.pb --out-dir ./ --output penguin.py --remat --max-costly-ops=2 --max-live-in-size=5 --max-remat-chain-size=10 --max-mem-multiple=1.8 --min-def-use-distance=500 --remat-policy=transformer --allow-same-pass-remat=true --split-abc --layers-per-module=1 --emit-tensor-level-dropout-ops --emit-tensor-level-rng-ops --native-to-custom-softmax --partitioner-opts='--transformer'
+------------ 
+Reported stdout: 
+DEBUG: needsModular? No. macCnt 0
+INFO: Switching to single-module compile. PrePartitionPipe skipped.
+INFO: Found memory bound graph
+INFO: Number of Native SoftmaxDx's detected and replaced: 0
+INFO: Number of Native Softmax's detected and replaced: 0
+Replaced 0 dropout sequences with OffloadedDropout
+INFO: HloMacCount has found 0
+INFO: Traffic has found 182891677
+INFO: AIF 0
+HLO Ops used in computation: add all-reduce broadcast compare concatenate constant copy divide get-tuple-element multiply parameter reduce reshape select sqrt subtract tuple 
+2025-01-07 14:44:53.575264: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %concatenate.674 = concatenate(%reshape.633, %reshape.634, %reshape.635, %reshape.636, %reshape.637, %reshape.638, %reshape.639, %reshape.640, %reshape.641, %reshape.642, %reshape.643, %reshape.644, %reshape.645, %reshape.646, %reshape.647, %reshape.648, %reshape.649, %reshape.650, %reshape.651, %reshape.652, %reshape.653, %reshape.654, %reshape.655, %reshape.656, %reshape.657, %reshape.658, %reshape.659, %reshape.660, %reshape.661, %reshape.662, %reshape.663, %reshape.664, %reshape.665, %reshape.666, %r... to 512 characters in the compiler's debug metadata
+2025-01-07 14:44:53.575951: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %all-reduce.803 = all-reduce(%multiply.761, %multiply.759, %multiply.757, %multiply.755, %multiply.753, %multiply.751, %multiply.749, %multiply.747, %multiply.745, %multiply.743, %multiply.741, %multiply.739, %multiply.737, %multiply.735, %multiply.733, %multiply.731, %multiply.729, %multiply.727, %multiply.725, %multiply.723, %multiply.721, %multiply.719, %multiply.717, %multiply.715, %multiply.713, %multiply.711, %multiply.709, %multiply.707, %multiply.705, %multiply.703, %multiply.701, %multiply.699, ... to 512 characters in the compiler's debug metadata
+2025-01-07 14:44:53.575971: W hilo/hlo2penguin/utils/DumpDebugInfo.cc:51] Truncating long HLO operator name %tuple.1 = tuple(%add.1039, %add.1081, %add.1123, %add.1165, %add.1207, %add.1249, %add.1291, %add.1333, %add.1375, %add.1417, %add.1459, %add.1501, %add.1543, %add.1585, %add.1627, %add.1669, %add.1711, %add.1753, %add.1795, %add.1837, %add.1879, %add.1921, %add.1963, %add.2005, %add.2047, %add.2089, %add.2131, %add.2173, %add.2215, %add.2257, %add.2299, %add.2341, %add.2383, %add.2425, %add.2467, %add.2509, %add.2551, %add.2593, %add.2635, %add.2677, %add.2719, %constant.265, %sqrt.683, %add.1024, %add... to 512 characters in the compiler's debug metadata
+Invoking RemoveOptimizationBarriers pass
+2025-01-07 14:44:53.587275: F hilo/MLIRPasses/Transforms/VerifyAliasing.cc:125] false && "Read after write on aliased input/output buffer."
+
+------------ 
+Reported stderr: 
+None
+------------                       
+Import of the HLO graph into the Neuron Compiler has failed.                       
+This may be caused by unsupported operators or an internal compiler error.                       
+More details can be found in the error message(s) above.
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "neuronxcc/driver/CommandDriver.py", line 352, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand_in_process
+  File "neuronxcc/driver/CommandDriver.py", line 347, in neuronxcc.driver.CommandDriver.CommandDriver.run_subcommand
+  File "neuronxcc/driver/CommandDriver.py", line 111, in neuronxcc.driver.CommandDriver.handleError
+  File "neuronxcc/driver/GlobalState.py", line 102, in neuronxcc.driver.GlobalState.FinalizeGlobalState
+  File "neuronxcc/driver/GlobalState.py", line 82, in neuronxcc.driver.GlobalState._GlobalStateImpl.shutdown
+  File "/usr/lib/python3.10/shutil.py", line 715, in rmtree
+    onerror(os.lstat, path, sys.exc_info())
+  File "/usr/lib/python3.10/shutil.py", line 713, in rmtree
+    orig_st = os.lstat(path)
+FileNotFoundError: [Errno 2] No such file or directory: '/tmp/no-user/neuroncc_compile_workdir/eb8c59e7-8e37-4ec2-80c9-43abb7d60b2c/neuronxcc-94ew1sgw'

neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_15940668397087814144+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8088b253d25f4710353508b7082528feb00434a6a1aea3e90767cce42e662b04
+size 138992

neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1"]

neuronxcc-2.16.345.0+69131dd3/MODULE_16715920953930297407+e30acd3a/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7590a741d6e1160f42eff0d8598ab24c5481dfe0c1256867ee875e6753df91a
+size 122498

neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/compile_flags.json

@@ -0,0 +1 @@
+["--target=trn1", "--model-type=transformer"]

neuronxcc-2.16.345.0+69131dd3/MODULE_17194828158773761536+3cc9a3cb/model.hlo_module.pb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:152764326e8a64a6ea303060d3c1faf961b75b60082da6743a7cd4bbab6f456d
+size 170704