使用python在pyspark中运行sql查询时出错

您好,我编写了一个代码,其中我要从TERR.txt文件创建数据帧。现在,我正在尝试运行sql查询,但出现了一些错误。

您能帮我解决这个问题吗?

代码:

from pyspark.context import SparkContext
from pyspark.sql import HiveContext
from pyspark.sql import SQLContext
from pyspark.sql.functions import *
from pyspark.sql.window import Window
from pyspark.sql.session import SparkSession
from pyspark.sql import Row
from pyspark.sql import DataFrame
from pyspark.context import SparkConf

conf = SparkConf()

sc = SparkContext(conf=conf)
spark = SparkSession(sc)
sqlContext = SQLContext(sc)
sc.setLogLevel("ERROR")

df = sqlContext.read.format('csv').option("delimiter","|").options(header='true',multiline = 'true',escape = '\"',inferSchema='false').load("/test/TERR.txt")

print(df)
print(df.show())

query = "select a.idasnmitory_id,a.dscrptnasnmitory_desc,a.nmasterr_nm,a.pstn_type,a.parnt_terrasparnt_nm_id,b.nmasparnt_terr_nm,a.start_dt,a.type,CASE WHEN substr(a.nm,1,6) IN ('105-30','105-31','105-32','105-41','105-42','105-43','200-CD','200-CG','200-CO','200-CP','200-CR','200-DG') THEN 'jbi' WHEN substr(a.nm,6) IN ('100-SC','105-05','105-06','105-07','105-08','105-13','105-71','105-72','105-73') THEN 'JP' WHEN substr(a.nm,6) IN ('103-16') THEN 'JT' WHEN substr(a.nm,6) IN ('105-51','200-HA','200-HF','200-HT','105-HT') THEN 'JSA' WHEN substr(a.nm,6) IN ('105-61','200-PR') THEN 'PR' WHEN substr(a.nm,3) IN ('302') THEN 'Canada-MEM' WHEN substr(a.nm,3) IN ('301') THEN 'Canada-MSL'ELSE'Unspecified'END AS DEPARTMENT,6) IN ('105-06','105-08') THEN 'CVMMSL' WHEN substr(a.nm,'105-13') THEN 'CVMCSS' WHEN substr(a.nm,6) IN ('105-41','200-CD') THEN 'DermMSL' WHEN substr(a.nm,6) IN ('105-42','200-CG') THEN 'GastroMSL' WHEN substr(a.nm,6) IN ('105-31') THEN 'HemeoncMSL' WHEN substr(a.nm,6) IN ('200-DG') THEN 'ImmMD' WHEN substr(a.nm,6) IN ('103-16') THEN 'IDMSL' WHEN substr(a.nm,6) IN ('200-CP') THEN 'ImmOps' WHEN substr(a.nm,6) IN ('105-05','105-73') THEN 'NeuroMSL' WHEN substr(a.nm,'200-CO') THEN 'OncMSL' WHEN substr(a.nm,'200-PR') THEN 'PuertoRicoMSL' WHEN substr(a.nm,6) IN ('105-43','200-CR') THEN 'RheumMSL' WHEN substr(a.nm,'200-HF') THEN 'RWVEField' WHEN substr(a.nm,6) IN ('105-32') THEN 'SolidTumorMSL' WHEN substr(a.nm,6) IN ('200-HT','105-HT') THEN 'RWVEPopHealth' WHEN substr(a.nm,6) IN ('301-PC') THEN 'Canada-PCMSL' WHEN substr(a.nm,6) IN ('301-VR') THEN 'Canada-VR/ONCMSL' WHEN substr(a.nm,6) IN ('301-SO') THEN 'Canada-Hematology(Myeloid)MSL' WHEN substr(a.nm,6) IN ('301-ON') THEN 'Canada-Hematology(Lymphoid)MSL' WHEN substr(a.nm,6) IN ('301-IP') THEN 'Canada-CNSMSL' WHEN substr(a.nm,6) IN ('301-RD') THEN 'Canada-RheumMSL' WHEN substr(a.nm,6) IN ('301-IB') THEN 'Canada-GastroMSL' WHEN substr(a.nm,6) IN ('301-DE') THEN 'Canada-DermMSL' WHEN substr(a.nm,6) IN ('301-SE') THEN 'Canada-BiologicsMSL' WHEN substr(a.nm,6) IN ('302-PC') THEN 'Canada-PCMEM' WHEN substr(a.nm,6) IN ('302-VR') THEN 'Canada-VR/ONCMEM' WHEN substr(a.nm,6) IN ('302-SO') THEN 'Canada-Hematology(Myeloid)MEM' WHEN substr(a.nm,6) IN ('302-ON') THEN 'Canada-Hematology(Lymphoid)MEM' WHEN substr(a.nm,6) IN ('302-IP') THEN 'Canada-CNSMEM' WHEN substr(a.nm,6) IN ('302-RD') THEN 'Canada-RheumMEM' WHEN substr(a.nm,6) IN ('302-IB') THEN 'Canada-GastroMEM' WHEN substr(a.nm,6) IN ('302-DE') THEN 'Canada-DermMEM' WHEN substr(a.nm,6) IN ('302-SE') THEN 'Canada-BiologicsMEM'ELSE'Unspecified'END AS FRANCHISE from outbound.terr a left outer joIN outbound.terr b on a.parnt_terr=b.id"

df.createOrReplaceTempView("terr")

df_output = spark.sql(query)

错误:

 Traceback (most recent call last):
  File "<stdin>",line 1,in <module>
  File "/usr/lib/spark/python/pyspark/sql/session.py",line 710,in sql
    return DataFrame(self._jsparkSession.sql(sqlQuery),self._wrapped)
  File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py",line 1257,in __call__
  File "/usr/lib/spark/python/pyspark/sql/utils.py",line 69,in deco
    raise AnalysisException(s.split(': ',1)[1],stackTrace)
pyspark.sql.utils.AnalysisException: u"Table or view not found: `outbound`.`terr`; line 1 pos 2791;\n'Project
ttzzhhssjjnn 回答:使用python在pyspark中运行sql查询时出错

暂时没有好的解决方案,如果你有好的解决方案,请发邮件至:iooj@foxmail.com
本文链接:https://www.f2er.com/3117478.html

大家都在问