I want to get the weekday from the list passing index into the already defined list, but index value will come from a dataframe column.
Issue is in last line where it is showing list indices must be integers or slices, not Column Any idea how to pass the indices to list from dataframe column value? thanks in advance...
Here is my code
from pyspark.sql.functions import to_date, col, lit,current_date,datediff,date_add,dayofweek
from pyspark.sql.types import IntegerType
rdd = sc.parallelize(\[(1,)\])
df = rdd.toDF('Dummy int')
df = df.withColumn("Start Date",to_date(lit('01/08/2023'),'dd/mm/yyyy'))\
.withColumn("End Date",current_date())\
.withColumn("Days",datediff(col("End Date"),col("Start Date")))
no_of_days = df.collect()\[0\]\[3\]
df.show()
weekdayList = \['Sunday','Monday','Tuesday','Wednesday','Thursday','Friday','Saturday'\]
df_days = spark.range(no_of_days).alias("no_of_days")
df_days.show(2)
df_days = df_days.join(df).withColumn("Date",date_add(col("Start Date"),col("id").cast("int")))\
.withColumn("Weekday",dayofweek(col("Date")))\
.withColumn("WeekdayStr",weekdayList\[col("Weekday").cast(IntegerType())%7\])