一、

二、代码

##粗召回数据处理
pandas_df = pd.read_csv("./clothes_women_1356.txt",sep='\t',dtype=str)

schema = StructType([StructField("item_sku_id", StringType(), True)
                   ,StructField("item_first_cate_cd", StringType(), True)
                    ,StructField("item_first_cate_name", StringType(), True)
                    ,StructField("item_second_cate_cd", StringType(), True)
                    ,StructField("item_second_cate_name", StringType(), True)
                    ,StructField("item_third_cate_cd", StringType(), True)
                    ,StructField("item_third_cate_name", StringType(), True)
                     ,StructField("sku_name", StringType(), True)
                   ,StructField("pic", StringType(), True)])

spark_df = spark.createDataFrame(pandas_df, schema=schema)
spark_df.registerTempTable("match_fs")
xtl_df = get_recall_data(dt)
xtl_df = xtl_df.repartition(500)

如果虚拟环境已经配置好了tf或者pytorch环境可以直接调用

一、先测试环境是否可用

一般路径：/home/admin/anaconda3/envs/py36_tf2_fj/bin/python
注意测试的时候在bin 目录下不能用 python 需要使用./python

1.tf框架

import tensorflow as tf
tf.test.is_gpu_available()
结果如果为True代表tf环境可用

2.torch框架

import torch
torch.cuda.is_available()

sudo docker run -d --name wechat --device /dev/snd --ipc="host" \
 -v /tmp/.X11-unix:/tmp/.X11-unix \
 -v $HOME/WeChatFiles:/WeChatFiles \
 -e DISPLAY=unix$DISPLAY \
 -e XMODIFIERS=@im=ibus \
 -e QT_IM_MODULE=ibus \
 -e GTK_IM_MODULE=ibus \
 -e AUDIO_GID=`getent group audio | cut -d: -f3` \
 -e GID=`id -g` \
 -e UID=`id -u` \
bestwu/wechat

source wechat.sh

docker start wechat
docker stop wechat
sudo docker rm -f wechat

http://shiyuefei.top/2020/09/01/CentOS%E5%AE%89%E8%A3%85%E5%BE%AE%E4%BF%A1/
https://www.jianshu.com/p/2a603db77195

https://zhuanlan.zhihu.com/p/379416038
https://zhuanlan.zhihu.com/p/323723229
https://www.jianshu.com/p/2a603db77195

SELECT
upc_y,
standard_id
FROM
table lateral VIEW explode(split(upc, ';')) t1 AS upc_y

注意这里对特殊字符要进行转义： '#\$%#' ';'
(1)有的符号是特殊字符，有的不是
'''

 select 
     sku_name
     sku_name_cleans,       
 from aa
 lateral VIEW explode(split(sku_name, '#\\\$%#')) t1 AS sku_name_cleans
 where 
      dt='2021-08-04'

'''

一、建表

CREATE TABLE a like b
查看hive建表语句：show create table tablename;

二、复制数据

set hive.exec.dynamic.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;
insert into table A partition (dt,data_type)
select * from B WHERE dt=sysdate(-1)

三、查看表结构

describe tablename; 简写：desc tablename;

四、排序

SELECT

*

FROM

(
 SELECT
    *,
    row_number() over(PARTITION by cid ORDER by rand() ) as rank
FROM
    table
WHERE
    dt ='2021-09-26'

 ) a  
 WHERE a.rank <101