dataframe常用操作方式

下边用示例代码，给大家做个演示，并针输出结果截图。

from pyspark import SparkContext,SparkConf 

from pyspark.sql.session import SparkSession 

from pyspark.sql.types import StructField, StructType, StringType 

if __name__ == '__main__':
    spark = SparkSession.builder.master("local").appName("SparkOnHive").getOrCreate()#.enableHiveSupport()   

    schema = StructType([ 

        # true代表不为空  

        StructField("WebsiteID", StringType(), True),  

        StructField("Url", StringType(), True),  

        StructField("IsDeleted", StringType(), True),  

        StructField("IsThirdPartySaler", StringType(), True),  

        StructField("JobHistory", StringType(), True),  

        StructField("RowKey", StringType(), True)
    ])
    df = spark.read.csv('file:///Users/jasongao/Documents/tmp/hbase-0.csv',schema) 

    # 显示dataframe数据，show方法不加具体数值，默认显示20条数据  

    df.show() 

    # 显示特定列  

    df.select('WebsiteID','Url').show() 

    # 按指定字段做groupby 操作  

    df.groupby('WebsiteID').count().show() 

    # 精确查询  

    df.filter(df['Url']=='item.jd.com/100000181227.html').show() 

    # 模糊查询，会显示所有数据，在最后补增列显示匹配结果  

    df.select('WebsiteID','Url',df['Url'].like('%181227%')).show() 

    # sql 查询方式，使查询更容易理解  

    df.registerTempTable('Product')
    spark.sql("select * from Product where Url ='item.jd.com/100000181227.html'").show() 



 spark.stop()