Oozie Coordinators in Prod environment are automatically getting killed with below logs:
Till 27th Aug Everything has run fine, on 28th Aug 09:00 the job is scheduled to run but job gets killed 20mins before scheduled run.
2023-08-27 10:06:00,005 TRACE CallableQueueService:529 - SERVER[hdp0117.global] USER[myuser] GROUP[-] TOKEN[] APP[app-myapp] JOB[0033644-230729144752906-oozie-oozi-C] ACTION[] poll(): [coord_action_input_0031677-230627122031098-oozie-oozi-C@1124], from P[1]
2023-08-28 08:40:39,360 TRACE CoordKillXCommand:529 - SERVER[0116.global] USER[myuser] GROUP[-] TOKEN[] APP[app-app-myapp] JOB[0033644-230729144752906-oozie-oozi-C] ACTION[-] Precondition check for command [coord_kill] key [0033644-230729144752906-oozie-oozi-C]
2023-08-28 08:40:39,360 DEBUG CoordKillXCommand:526 - SERVER[0116.global] USER[myuser] GROUP[-] TOKEN[] APP[app-app-myapp] JOB[0033644-230729144752906-oozie-oozi-C] ACTION[-] Execute command [coord_kill] key [0033644-230729144752906-oozie-oozi-C]
2023-08-28 08:40:39,361 DEBUG CoordKillXCommand:526 - SERVER[0116.global] USER[myuser] GROUP[-] TOKEN[] APP[app-myapp] JOB[0033644-230729144752906-oozie-oozi-C] ACTION[-] Killed coord actions for the coordinator=[0033644-230729144752906-oozie-oozi-C]
End Time is configured as 3000-01-01.
<coordinator-app name="${appName}"
frequency="${jobFrequency}"
start="${startTime}"
end="${endTime}"
timezone="UTC"
xmlns="uri:oozie:coordinator:0.5">
<controls>
<concurrency>1</concurrency>
<throttle>1</throttle>
</controls>
<datasets>
<dataset name="output_dataset" frequency="${coord:days(1)}"
initial-instance="${initialDatasetInstance}" timezone="UTC">
<uri-template>
${oozieTriggerFileLocation}/${moduleName}/${YEAR}-${MONTH}-${DAY}/
</uri-template>
</dataset>
</datasets>
<output-events>
<data-out name="output_events" dataset="output_dataset">
<instance>${coord:current(0)}</instance>
</data-out>
</output-events>
<action>
<workflow>
<app-path>${codePath}</app-path>
<configuration>
<property>
<name>today</name>
<value>${coord:formatTime(coord:nominalTime(), "yyyy-MM-dd")}</value>
</property>
<property>
<name>yesterday</name>
<value>${coord:formatTime(coord:dateOffset(coord:nominalTime(), -1, 'DAY'),'yyyy-MM-dd')}</value>
</property>
<property>
<name>nominalTime</name>
<value>${coord:nominalTime()}</value>
</property>
</configuration>
</workflow>
</action>
</coordinator-app>