# -*- coding: utf-8 -*-
"""
 * target ： 爬朱雯轩的所有短视频
 * @Date ： 2020-09-29 13:03
 * @Auth ： xiaoshuai.zhu
 * @File ：爬取抖音用户视频.py
 * @IDE ：PyCharm
 * @Version 1.0
"""

import os
import requests
import random
import time
import urllib3

def timestamp():
    '''
    生成时间戳
    :return:
    '''
    min_t = int(time.time())
    max_t = int(time.time() * 1000)
    return str(min_t), str(max_t)

def fabu_time(t):
    '''
    将时间戳转换成时间格式
    :param t:
    :return:
    '''
    timeArray = time.localtime(t)
    fabu_time = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
    return fabu_time

def save(dict):
    '''
    传入字典保存数据
    :param dict:
    :return:
    '''
    # 有些短视频中没有标题导致我们再保存数据时会覆盖，所以我们对没有标题的短视频进行命名
    num = str(random.randint(0, 100))
    if dict['desc'] == '':
        name = dict['user_name'] + '的作品{}'.format(num) + '.mp4'
    else:
        name = dict['desc'] + '.mp4'
    video_url = dict['video_url']
    res = requests.get(video_url)
    file_name = 'D:/Ponfey/Object/Study/抖音短视频/抖音短视频_' + dict['user_name']
    if os.path.exists(file_name):
        with open(file_name + '/' + name, 'wb') as f:
            f.write(res.content)
            print('{}下载完成！'.format(name))
    else:
        os.mkdir(file_name)
        with open(file_name + '/' + name, 'wb') as f:
            f.write(res.content)
            print('{}下载完成！'.format(name))

def get_data(url):
    '''
    根据请求url获取数据
    :param url:
    :return:
    data:list数据
    max_cursor:下次请求需要携带的参数
    '''
    while True:
        min_t, max_t = timestamp()
        headers = {'Connection': 'keep-alive',
                   'Cookie': 'd_ticket=4c5b44a063bf078fae71bffec25ddad8ca4ea; odin_tt=6450ec41def6afd0731d426731b508fcf43650505f50460244a368ac5847f0d1cbf6747a7ad4e89fa3b0f0e15754002e; sid_guard=a6001040b2e52133062ca1e743097c06%7C1588771917%7C5183999%7CSun%2C+05-Jul-2020+13%3A31%3A56+GMT; uid_tt=de424823d2132aab28fa581760578e06; uid_tt_ss=de424823d2132aab28fa581760578e06; sid_tt=a6001040b2e52133062ca1e743097c06; sessionid=a6001040b2e52133062ca1e743097c06; sessionid_ss=a6001040b2e52133062ca1e743097c06; install_id=4019449375779358; ttreq=1$4e594dc75197827452871cc4798ece9879de8c47',
                   'X-SS-REQ-TICKET': '1588773397905',
                   'X-Tt-Token': '00a6001040b2e52133062ca1e743097c06cdb862c09a8bd8d850534d8a1ddf1bcda1297dfe483093add3539ae384dd657413',
                   'sdk-version': '1',
                   'X-SS-DP': '1128',
                   'x-tt-trace-id': '00-ea46271a0d9c7aafc2c2647f8e990468-ea46271a0d9c7aaf-01',
                   'User-Agent': 'com.ss.android.ugc.aweme/100901 (Linux; U; Android 5.1.1; zh_CN; SM-N960F; Build/JLS36C; Cronet/TTNetVersion:8109b77c 2020-04-15 QuicVersion:0144d358 2020-03-24)',
                   'Accept-Encoding': 'gzip',
                   'X-Gorgon': '0404b8014001a0ff70a608a72b1c1754b85bab86f700fceac8f4',
                   'X-Khronos': min_t,
                   'x-common-params-v2': 'os_api=22&device_platform=android&device_type=SM-N960F&iid=4019449375779358&version_code=100900&app_name=aweme&openudid=9a7fc881896f46bf&device_id=2752811979515463&os_version=5.1.1&aid=1128&channel=tengxun_new&ssmix=a&manifest_version_code=100901&dpi=240&cdid=1421f378-62fa-44ad-af4f-49d33aa7a58a&version_name=10.9.0&resolution=720*1280&language=zh&device_brand=samsung&app_type=normal&ac=wifi&update_version_code=10909900&uuid=355757648741243'}

        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        response = requests.get(url, headers=headers, verify=False).json()
        try:
            max_cursor = response['max_cursor']
            if not response['aweme_list']:
                pass
            else:
                data = response['aweme_list']
                return data, max_cursor
        except:
            data = None
            max_cursor = None
            return data, max_cursor

def data_parse(data):
    '''
    用户短视频数据解析
    :param data:
    :return:
    '''
    production_list = []
    for i in data:
        production_dict = {
            'user_name': i['author']['nickname'],
            'desc': i['desc'],
            'time': fabu_time(i['create_time']),
            'music_author': i['music']['author'],
            'music_name': i['music']['title'],
            'video_url': i['video']['play_addr_lowbr']['url_list'][0]
        }
        production_list.append(production_dict)
    return production_list

# headers数据参数不正确请求不到数据
def attention_me_uid(url):
    '''
    获取我关注的抖音用户的uid与下一页请求锁需要的参数
    :return:
    '''
    while True:
        min_t, max_t = timestamp()
        headers = {'Connection': "keep-alive",
                   'Cookie': 'd_ticket=4c5b44a063bf078fae71bffec25ddad8ca4ea; odin_tt=6450ec41def6afd0731d426731b508fcf43650505f50460244a368ac5847f0d1cbf6747a7ad4e89fa3b0f0e15754002e; sid_guard=a6001040b2e52133062ca1e743097c06%7C1588771917%7C5183999%7CSun%2C+05-Jul-2020+13%3A31%3A56+GMT; uid_tt=de424823d2132aab28fa581760578e06; uid_tt_ss=de424823d2132aab28fa581760578e06; sid_tt=a6001040b2e52133062ca1e743097c06; sessionid=a6001040b2e52133062ca1e743097c06; sessionid_ss=a6001040b2e52133062ca1e743097c06; install_id=4019449375779358; ttreq=1$4e594dc75197827452871cc4798ece9879de8c47',
                   'X-SS-REQ-TICKET': '1588777054732',
                   'X-Tt-Token': '00a6001040b2e52133062ca1e743097c06cdb862c09a8bd8d850534d8a1ddf1bcda1297dfe483093add3539ae384dd657413',
                   'sdk-version': '1',
                   'X-SS-DP': '1128',
                   'x-tt-trace-id': '00-ea7df3860d9c7aafc2c2647233a00468-ea7df3860d9c7aaf-01',
                   'User-Agent': 'com.ss.android.ugc.aweme/100901 (Linux; U; Android 5.1.1; zh_CN; SM-N960F; Build/JLS36C; Cronet/TTNetVersion:8109b77c 2020-04-15 QuicVersion:0144d358 2020-03-24)',
                   'Accept-Encoding': 'gzip',
                   'X-Gorgon': '0404b8014001fc330e5d08a72b1c1754b85bab86f700fca29ee3',
                   'X-Khronos': min_t,
                   'x-common-params-v2': 'os_api=22&device_platform=android&device_type=SM-N960F&iid=4019449375779358&version_code=100900&app_name=aweme&openudid=9a7fc881896f46bf&device_id=2752811979515463&os_version=5.1.1&aid=1128&channel=tengxun_new&ssmix=a&manifest_version_code=100901&dpi=240&cdid=1421f378-62fa-44ad-af4f-49d33aa7a58a&version_name=10.9.0&resolution=720*1280&language=zh&device_brand=samsung&app_type=normal&ac=wifi&update_version_code=10909900&uuid=355757648741243'}
        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
        response = requests.get(url, headers=headers, verify=False).json()
        print(response)
        try:
            max_time = response['max_time']
            if not response['followings']:
                pass
            else:
                data = response['followings']
                return data, max_time
        except:
            data = None
            max_cursor = None
            return data, max_cursor

# 将获取到的关注的抖音用户数据进行解析
def uid_parse(data):
    '''
    传入的是列表
    :param data:
    :return:
    '''
    user_list = []
    for i in data:
        user_dict = {
            'sec_uid': i['sec_uid']
        }
        user_list.append(user_dict)
    return user_list

# 第一次请求时携带的max_time,将max_time 传入后获取关注抖音用户列表
def first_user():
    min_t, max_t = timestamp()
    url = 'https://api3-normal-c-lq.amemv.com/aweme/v1/user/following/list/?user_id=86304636253&sec_user_id=MS4wLjABAAAAliUfImgLRYe1ih0ZL0_GQ3dzUAOGZ1JEInos9icA04w&max_time=0&count=20&offset=0&source_type=2&address_book_access=2&gps_access=1&vcd_count=0&vcd_auth_first_time=0&ts={}&host_abi=armeabi-v7a&_rticket={}&mcc_mnc=46007&'.format(
        min_t, max_t)
    headers = {'Connection': 'keep-alive',
               'Cookie': 'd_ticket=4c5b44a063bf078fae71bffec25ddad8ca4ea; odin_tt=6450ec41def6afd0731d426731b508fcf43650505f50460244a368ac5847f0d1cbf6747a7ad4e89fa3b0f0e15754002e; sid_guard=a6001040b2e52133062ca1e743097c06%7C1588771917%7C5183999%7CSun%2C+05-Jul-2020+13%3A31%3A56+GMT; uid_tt=de424823d2132aab28fa581760578e06; uid_tt_ss=de424823d2132aab28fa581760578e06; sid_tt=a6001040b2e52133062ca1e743097c06; sessionid=a6001040b2e52133062ca1e743097c06; sessionid_ss=a6001040b2e52133062ca1e743097c06; install_id=4019449375779358; ttreq=1$4e594dc75197827452871cc4798ece9879de8c47',
               'X-SS-REQ-TICKET': '1588780059093',
               'X-Tt-Token': '00a6001040b2e52133062ca1e743097c06cdb862c09a8bd8d850534d8a1ddf1bcda1297dfe483093add3539ae384dd657413',
               'sdk-version': '1', 'X-SS-DP': '1128',
               'x-tt-trace-id': '00-eaabcb4c0d9c7aafc2c26472519d0468-eaabcb4c0d9c7aaf-01',
               'User-Agent': 'com.ss.android.ugc.aweme/100901 (Linux; U; Android 5.1.1; zh_CN; SM-N960F; Build/JLS36C; Cronet/TTNetVersion:8109b77c 2020-04-15 QuicVersion:0144d358 2020-03-24)',
               'Accept-Encoding': 'gzip', 'X-Gorgon': '0404b8014001be0483eb08a72b1c1754b85bab86f700fc923f8b',
               'X-Khronos': '1588780059',
               'x-common-params-v2': 'os_api=22&device_platform=android&device_type=SM-N960F&iid=4019449375779358&version_code=100900&app_name=aweme&openudid=9a7fc881896f46bf&device_id=2752811979515463&os_version=5.1.1&aid=1128&channel=tengxun_new&ssmix=a&manifest_version_code=100901&dpi=240&cdid=1421f378-62fa-44ad-af4f-49d33aa7a58a&version_name=10.9.0&resolution=720*1280&language=zh&device_brand=samsung&app_type=normal&ac=wifi&update_version_code=10909900&uuid=355757648741243'}
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
    response = requests.get(url, headers=headers, verify=False).json()
    print(response)

# 一边循环获取关注的抖音用户的sec_uid一边对单个用户进行数据的爬取
def run():
    max_time = 1588778511
    while True:
        min_t, max_t = timestamp()
        url = 'https://api3-normal-c-lq.amemv.com/aweme/v1/user/following/list/?user_id=86304636253&sec_user_id=MS4wLjABAAAAliUfImgLRYe1ih0ZL0_GQ3dzUAOGZ1JEInos9icA04w&max_time={}&count=20&offset=0&source_type=1&address_book_access=2&gps_access=1&vcd_count=0&vcd_auth_first_time=0&ts={}4&host_abi=armeabi-v7a&_rticket={}&mcc_mnc=46007&'.format(
            max_time, min_t, max_t)
        print(url)
        data, max_time = attention_me_uid(url)
        print(data)
        if data is not None and max_time is not None:
            user_list = uid_parse(data)
            for user in user_list:
                sec_uid = user['sec_uid']
                user_get(sec_uid)
        else:
            print('您关注的抖音用户已爬完！')
            break

def user_get(sec_uid):
    '''
    传入目标抖音用户的sec_uid对单个抖音用户循环爬取所有短视频
    :param sec_uid:
    :return:
    '''
    max_cursor = 0
    while True:
        min_t, max_t = timestamp()
        url = 'https://api3-normal-c-lq.amemv.com/aweme/v1/aweme/post/?source=0&publish_video_strategy_type=0&max_cursor={}&sec_user_id={}&count=20&ts={}&host_abi=armeabi-v7a&_rticket={}&mcc_mnc=46007&'.format(
            max_cursor, sec_uid, min_t, max_t)
        data, max_cursor = get_data(url)
        if data is not None and max_cursor is not None:
            production_list = data_parse(data)
            for production in production_list:
                save(production)
        else:
            print('您喜欢的抖音用户已爬完！')
            break

if __name__ == '__main__':
    sec_uid = 'MS4wLjABAAAAqszdrdTPFmmpQXXvMMxrx2Dmj-4sdMQYOIcduwNAfjM' #  朱雯轩的个人主页
    user_get(sec_uid)