编写SparkSQL语句的最佳方法是什么？
对此类陈述的任何建议：
import pyspark.sql.functions as psf
df_final = df_01\
.select(df_01['*'],
psf.when(df_01.record_two>\
df_01.record_three,
df_01.record_two)
.otherwise(df_01.record_three)\
.alias("some_new_alias"))\
.where("some_field > 1000")\
.where((df_01.record_one.isin(df_red01.record_one)==False) |\
(df_01.record_one.isin(df_blue01.record_one)==False) |\
(df_01.record_one.isin(df_violet01.record_one)==False) |\
(df_01.record_one.isin(df_green01.record_one)==False) |\
(df_01.record_one.isin(df_black01.record_one)==False) |\
(df_01.record_one.isin(df_white01.record_one)==False) |\
(df_01.record_one.isin(df_red02.record_one)==False) |\
(df_01.record_one.isin(df_blue02.record_one)==False) |\
(df_01.record_one.isin(df_violet02.record_one)==False) |\
(df_01.record_one.isin(df_green02.record_one)==False) |\
(df_01.record_one.isin(df_black02.record_one)==False) |\
(df_01.record_one.isin(df_white02.record_one)==False) |\
(df_01.record_one.isin(df_blue03.record_one)==False) |\
(df_01.record_one.isin(df_violet03.record_one)==False) |\
(df_01.record_one.isin(df_green03.record_one)==False) |\
(df_01.record_one.isin(df_black03.record_one)==False) |\
(df_01.record_one.isin(df_violet04.record_one)==False) |\
(df_01.record_one.isin(df_green04.record_one)==False) |\
(df_01.record_one.isin(df_violet04.record_one)==False))\
.select("record_one", "some_new_alias")
df_another_test_frame = df_jibber01\
.select(df_jibber01.field01,
df_jibber01.field02,
df_jibber01.field03,
df_jibber01.field04,
df_jibber01.field05,
df_jibber01.field06,
df_jibber01.field07,
df_jibber01.field08,
df_jibber01.field09,
psf.when(df_jibber01.field04 <= 100,
psf.round(2000*df_jibber01.field10/59, 10))\
.when(df_jibber01.field05 >= 1,
psf.round(2000*df_jibber01.field10/59, 10))
.when(df_jibber01.field06 >= 2,
psf.round(2000*df_jibber01.field10/59, 10))
.when(df_jibber01.field04 <= 3,
psf.round(20*df_jibber01.field10/59, 10))
.when(df_jibber01.field05 >= 4,
psf.round(20*df_jibber01.field10/59, 10))
.when(df_jibber01.field06 >= 5,
psf.round(20*df_jibber01.field10/59, 10))
.when(df_jibber01.field04 <= 6,
psf.round(9999*df_jibber01.field10/59, 10))
.when(df_jibber01.field05 >= 7,
psf.round(9999*df_jibber01.field10/59, 10))
.when(df_jibber01.field06 >= 8,
psf.round(9999*df_jibber01.field10/59, 10))
.otherwise(psf.round(9999*df_jibber01.field10/59, 10))\
.alias("field11")
)
有多个＆＃34;其中＆＃34;和＃34;当＆＃34;条件。有更清洁的写作方式吗？我有100多个这样的陈述。
任何建议都会有所帮助。