遠(yuǎn)程傳文件
- 從本地復(fù)制到遠(yuǎn)程
scp local_file remote_username@remote_ip:remote_folder
或者
scp local_file remote_username@remote_ip:remote_file
或者
scp local_file remote_ip:remote_folder
或者
scp local_file remote_ip:remote_file
- 從遠(yuǎn)程復(fù)制到本地
scp root@www.runoob.com:/home/root/others/music /home/space/music/1.mp3
scp -r www.runoob.com:/home/root/others/ /home/space/music/
讀文件
- spark.read.csv()
可以讀取csv、tsv、snappy壓縮文件等
from pyspark.sql import types
# 設(shè)置字段schema
schema = types.StructType([
types.StructField('id', types.LongType()),
types.StructField('tag', types.StringType())])
df = spark.read.csv(results_path, sep='\t', schema=schema)
# 或者自帶header
df = spark.read.csv(results_path, header=False, inferSchema=False, sep='\t')
寫(xiě)文件
- df.write.csv()
# 分區(qū)寫(xiě)文件
df.coalesce(1).write.csv(results_path, sep='\t', header=True, compression='none', mode='overwrite')
# 不分區(qū)寫(xiě)成單個(gè)文件
df.repartition(1).write.csv(results_path, sep='\t', header=False, compression='none', mode='overwrite')
列操作
- 保留列
df = df.select(['a', 'b'])
- 增加一個(gè)新列
# withColumn只能添加 df 已有列的變換
df = df.withColumn('a', col('b'))
- 刪除dataframe某些列
df = df.drop('a', 'b', 'c')
- 更改列名
from pyspark.sql.functions import col
# 方法一
df = df.withColumnRenamed('a', col('b'))
# 方法二
mapping = dict(zip(['_c0'], ['gid']))
test = test.select([col(c).alias(mapping.get(c, c)) for c in test.columns])
- 替換列的字符串并生成新列
new_df = df.withColumn("vv_new", regexp_replace(col("vv"), "na", "nan")).drop('vv')
- 按列filter過(guò)濾
df = df.filter(df.tag_name.like('%文化%')) # 保留某個(gè)匹配值
groupby
- 聚合統(tǒng)計(jì)操作
df_count = df.groupby('id').count() # groupby數(shù)量
df2 = df_count.filter(df_count['count']>=50)
- 按某列g(shù)roupby
id_list = vb_res.select('id').distinct().rdd.flatMap(lambda x: x).collect()
data = [vb_res.where(vb_res['id'] == id) for id in id_list]
print('id 數(shù)量：' + str(len(data)))
拼接兩個(gè)dataframe
- 直接append，前提是schema相同
a = spark.createDataFrame([(1, 'xxx'), (2, 'xxx')], ['photo_id', 'caption'])
b = spark.createDataFrame([(1, 'xxx'), (2, 'xxx')], ['photo_id', 'caption'])
c = a.unionAll(b)
c.show()
- 按字段join
spark.conf.set("spark.sql.crossJoin.enabled", "true")
new_df = df.join(filter_data, df['gid']==filter_data['gid_1'])
new_df = new_df.drop('gid_1')
new_df
類(lèi)型轉(zhuǎn)換
- sql CAST函數(shù)
CAST (movie_score AS int)
- spark cast
from pyspark.sql import types
df_after = df.select(col("movie_score"), col("movie_score").cast(types.LongType()))
寫(xiě)hive表
- 不追加，直接覆蓋xx.yy表
data.write.mode("overwrite").insertInto("xx.yy")
- sql insert 寫(xiě)hive表
# 使用普通的hive-sql寫(xiě)入分區(qū)表
spark.sql("""
insert overwrite table ai.da_aipurchase_dailysale_hive
partition (saledate)
select productid, propertyid, processcenterid, saleplatform, sku, poa, salecount, saledate
from szy_aipurchase_tmp_szy_dailysale distribute by saledate
""")
# 先將dataframe注冊(cè)成臨時(shí)表，然后通過(guò)sql的方式插入
df.createOrReplaceTempView("temp_tab")
spark.sql("insert into zz_table select * from temp_tab")
- saveAsTable()
# 不寫(xiě)分區(qū)表，只是簡(jiǎn)單的導(dǎo)入到hive表
df.write.saveAsTable("xx.yy", None, "overwrite", None)
# 在hive表已有的表xx.yy中追加記錄，按date分區(qū)
df.write.format("Hive").mode("append").saveAsTable("xx.yy", partitionBy='date')
- insertInto()
如果想要在不影響其他分區(qū)的情況下覆蓋某個(gè)指定分區(qū)的數(shù)據(jù)，可以用insertInto()
# 1.首先在SparkSession設(shè)置config
config("spark.sql.sources.partitionOverwriteMode", "DYNAMIC")
# 2.將列對(duì)其之后
data.write.format("Hive").mode("overwrite").insertInto(table_name)
參考：Spark——Spark覆蓋分區(qū)表中指定的分區(qū)
saveAsTable與insertInto的區(qū)別：
- saveAsTable——當(dāng)hive中已經(jīng)存在目標(biāo)表，無(wú)論SaveMode是append還是overwrite，不需要schema一樣，只要列名存在就會(huì)根據(jù)列名進(jìn)行匹配覆蓋數(shù)據(jù)
- insertInto——當(dāng)hive中存在目標(biāo)表時(shí)，無(wú)論SaveMode是append還是overwrite，需要當(dāng)前DF的schema與目標(biāo)表的schema必須一致，因?yàn)閕nsertInto插入的時(shí)候，是根據(jù)列的位置插入，而不是根據(jù)列的名字
UDF
- 解析json數(shù)據(jù)并生成一列新列 - by line
@udf('array<struct<a:bigint, b:float>>') # 注冊(cè)u(píng)df函數(shù)
def parse_dict_udf(s):
dic = json.loads(s)
return [{'a': int(a), 'b': d[a]} for a in dic]
df_transform = df.withColumn('b', explode(parse_dict_udf('a')))
- json數(shù)據(jù)解析 - by row
def convert_result_emb(row):
item = row.asDict()
result_dic = json.loads(item[u'result'].encode('utf-8'))
item['embedding'] = result_dic['embeding']
del item[u'result']
return Row(**item)
filter_data = spark.createDataFrame(data.rdd.map(convert_result_emb))
- 傳入多個(gè)參數(shù)
def get_text(title, name):
if not name: name = ''
if not title: title = ''
return title + name
func = udf(get_text, types.StringType()) # 注冊(cè)u(píng)df函數(shù)
df = df.withColumn("text", func(df.title, df.name))
- 傳入字典/tuple等特殊數(shù)據(jù)類(lèi)型
## 方法一
from pyspark.sql.types import StringType
from pyspark.sql.functions import udf
def translate(mapping):
def translate_(col):
return mapping.get(col)
return udf(translate_, StringType())
df = sc.parallelize([('DS', ), ('G', ), ('INVALID', )]).toDF(['key'])
mapping = {
'A': 'S', 'B': 'S', 'C': 'S', 'DS': 'S', 'DNS': 'S',
'E': 'NS', 'F': 'NS', 'G': 'NS', 'H': 'NS'}
df.withColumn("value", translate(mapping)("key"))
## 方法二 (Spark >= 2.0, Spark < 3.0)
from pyspark.sql.functions import col, create_map, lit
from itertools import chain
mapping_expr = create_map([lit(x) for x in chain(*mapping.items())])
df.withColumn("value", mapping_expr.getItem(col("key")))
## 方法二 (Spark >= 3.0)
from pyspark.sql.functions import col, create_map, lit
from itertools import chain
mapping_expr = create_map([lit(x) for x in chain(*mapping.items())])
df.withColumn("value", mapping_expr[col("key")]).show()
參考：
PySpark create new column with mapping from a dict
Pyspark-UDF函數(shù)的使用、UDF傳入多個(gè)參數(shù)、UDF傳出多個(gè)參數(shù)、傳入特殊數(shù)據(jù)類(lèi)型
- pyspark去重
# distinct
# reduceByKey
rdd.reduceByKey(lambda x,y:x)
# drop_duplicates
df.dropDuplicates([col_name1, col_name2])
df.drop_duplicates([col_name1, col_name2])
- pyspark排序
spark.createDataFrame(dfall).orderBy(desc('datestr')) # 降序
spark.createDataFrame(dfall).orderBy(df.datestr.desc()) # 降序
- 排序+去重
# 方法一：sort+drop_duplicates
df = df.sort(['movie_score'], ascending=False)
new_df = df.drop_duplicates(['item_id'])
# 方法二：開(kāi)窗函數(shù)
from pyspark.sql import Window
from pyspark.sql.functions import rank
window = Window.partitionBy(['item_id']).orderBy(['movie_score'])
df_1 = df.withColumn('rank', rank().over(window))
window = Window.partitionBy("col1").orderBy("datestr")
df_1 = df.withColumn('rank', rank().over(window)) # 保留所有排序
df_1 = df.withColumn('rank', rank().over(window)).filter(col('rank') == 1).drop('rank') # 保留第一行
- 對(duì)某列加和
# 方法一
df.groupBy().sum().collect()[0][0]
# 方法二
sum_number = df.agg({"a":"sum"}).collect()[0]
result = sum_number["sum(a)"]
- 生成libsvm格式文件
def to_libsvm(row):
item = row.asDict()
new_item = {}
feat_txt = ' '.join(['{}:{}'.format(i, t[1]) for i,t in enumerate(item.items())])
libsvm_txt = '{} {}'.format(item['label'], feat_txt)
new_item['_c0'] = libsvm_txt
return Row(**new_item)
final_df = spark.createDataFrame(new_df.rdd.map(to_libsvm))
OneHotEncoder
- 不使用pipeline
from pyspark.ml.feature import OneHotEncoder, StringIndexer
df = spark.createDataFrame([
(0, "a"),
(1, "b"),
(2, "c"),
(3, "a"),
(4, "a"),
(5, "c")
], ["id", "value"])
stringIndexer = StringIndexer(inputCol="value", outputCol="valueIndex")
model = stringIndexer.fit(df)
indexed = model.transform(df)
encoder = OneHotEncoder(inputCol="valueIndex", outputCol="valueIndexVec")
encoded = encoder.transform(indexed)
encoded.show()
- 使用pipeline
from pyspark.ml import Pipeline
from pyspark.ml.feature import OneHotEncoder, StringIndexer
from pyspark.sql.functions import col
# setHandleInvalid(“keep”) 防止無(wú)知新數(shù)據(jù)報(bào)錯(cuò)
stringIndexer = StringIndexer(inputCol="_c112", outputCol="_c112_indexed").setHandleInvalid("keep")
encoder = OneHotEncoder(inputCol="_c112_indexed", outputCol="poi_type_vec")
pipeline = Pipeline(stages=[stringIndexer, encoder])
model = pipeline.fit(geo_df)
transformed = model.transform(geo_df)
transformed.select('poi_type_vec').show(1, False)
Tips:
- row_number() | rank() | dense_rank() 的區(qū)別
row_number 1 2 3
rank 1 2 2
dense_rank 1 1 3
