- 穷举编程
- ccv编程
- 百度编程
- 谷歌编程
- gayhub编程
- guess编程
- no think
- 群友编程
- 小黄鸭调试法(Rubber Duck Debugging)
- 询问式撸代码
只需要 抛出 一个需求 代码大神来解决
代码人生:编织技术与生活的博客之旅
只需要 抛出 一个需求 代码大神来解决
通过Chrome与selenium 将网易云私信中歌曲放到一个歌单中,方便实时收听新歌。 我们可以直接用已登录的Chrome(保存有网易云网站登录态),模拟用户操作,收藏新歌。

def init_driver():
executable_path = "chromedriver"
if not os.path.exists("chromedriver.exe"): # 无驱则需要下载
# 本地项目 复制驱动
_path = os.path.abspath(__file__)
if not "crawler_set" in _path:
root_path = _path[:_path.index("crawler_set") + len("crawler_set")]
executable_path = os.path.join(root_path, "driver", "chromedriver.exe")
else: # 单个文件 下载驱动
url = "https://raw.githubusercontent.com/AngusWG/crawler_set/master/driver/chromedriver.exe"
proxies = {
"http": "socks5://127.0.0.1:1080",
"https": "socks5://127.0.0.1:1080"
}
r = requests.get(url, proxies=proxies)
with open("chromedriver.exe", "wb") as code:
content_size = int(r.headers['content-length']) # 内容体总大小
for data in tqdm(iterable=r.iter_content(1024), total=content_size, unit="k", desc="下载驱动"):
code.write(data)
user_cookies = "".join([os.path.expanduser('~'), r"\AppData\Local\Google\Chrome\User Data"])
option = webdriver.ChromeOptions()
option.add_argument("--user-data-dir={}".format(user_cookies)) # 设置成用户自己的数据目录
try:
driver = webdriver.Chrome(executable_path, options=option)
driver.implicitly_wait(5)
return driver
except WebDriverException:
print("请先关掉所有的Chrome")
exit(-2)
def save_song(url):
print("[{}] start".format(url), end=" ")
driver.get(url)
driver.switch_to.frame("contentFrame")
title = driver.find_element_by_xpath('//div[contains(@class, "tit")]').text
for word in shielding_words:
if word in title:
print("{} 因 {} 已忽略".format(title, word))
return
driver.find_element_by_xpath('//*[contains(text(), "收藏")]').click()
driver.find_element_by_xpath('//*[contains(text(), "{}")]'.format(song_dir)).click()
print(title)
#!/usr/bin/python3
# encoding: utf-8
# @Time : 2019/12/19 9:36
# @author : zza
# @Email : 740713651@qq.com
# @File : 将网易云的私信音乐整理.py
"""
该脚本会保存每天新私信里最后几首歌曲到tmp_save_dir
1.在Chrome上登录自己的网易云帐号t
2.创建tmp_save_dir
然后运行脚本
"""
import os
import requests
from selenium import webdriver
from selenium.common.exceptions import WebDriverException, NoSuchElementException
from tqdm import tqdm
song_dir = "tmp_save_dir"
# 屏蔽关键词
shielding_words = ['伴奏']
def init_driver():
executable_path = "chromedriver"
if not os.path.exists("chromedriver.exe"): # 无驱则需要下载
# 本地项目 复制驱动
_path = os.path.abspath(__file__)
if not "crawler_set" in _path:
root_path = _path[:_path.index("crawler_set") + len("crawler_set")]
executable_path = os.path.join(root_path, "driver", "chromedriver.exe")
else: # 单个文件 下载驱动
url = "https://raw.githubusercontent.com/AngusWG/crawler_set/master/driver/chromedriver.exe"
proxies = {
"http": "socks5://127.0.0.1:1080",
"https": "socks5://127.0.0.1:1080"
}
r = requests.get(url, proxies=proxies)
with open("chromedriver.exe", "wb") as code:
content_size = int(r.headers['content-length']) # 内容体总大小
for data in tqdm(iterable=r.iter_content(1024), total=content_size, unit="k", desc="下载驱动"):
code.write(data)
user_cookies = "".join([os.path.expanduser('~'), r"\AppData\Local\Google\Chrome\User Data"])
option = webdriver.ChromeOptions()
option.add_argument("--user-data-dir={}".format(user_cookies)) # 设置成用户自己的数据目录
try:
driver = webdriver.Chrome(executable_path, options=option)
driver.implicitly_wait(5)
return driver
except WebDriverException:
print("请先关掉所有的Chrome")
exit(-2)
def get_private_detail():
# 点击私信后
driver.get("https://music.163.com/#/msg/m/private")
driver.switch_to.frame("contentFrame")
new_msg_items = driver.find_elements_by_xpath('//i[@class="u-bub"]/b[@class="f-alpha"]/..//parent::*//a')
private_detail_url_dict = dict()
for i in new_msg_items:
_, singer_id = i.get_attribute("href").split("?")
uri = "https://music.163.com/#/msg/m/private_detail?" + singer_id
msg_num = int(i.find_element_by_xpath("../i/em").text)
private_detail_url_dict[uri] = msg_num
return private_detail_url_dict
def get_song_url_from_album_set(url):
song_set = set()
# 歌曲页面保存
driver.get(url)
driver.switch_to.frame("contentFrame")
url_list = driver.find_elements_by_xpath('//a[contains(@href, "/song?id")]')
for item in url_list:
if "伴奏" in item.text:
break
_, song_id = item.get_attribute("href").split("?id=")
song_set.add("https://music.163.com/#/song?id=" + song_id)
song_name = item.find_element_by_xpath("./b").get_attribute("title")
print(song_name, end=" ")
return song_set
def get_song_url_from_private_detail(url, msg_num):
album_set = set()
song_set = set()
# 歌曲页面保存
driver.get(url)
driver.switch_to.frame("contentFrame")
url_list = driver.find_elements_by_xpath('//div[contains(@class,"itemleft")]')[-msg_num:]
for item in url_list:
try:
i = item.find_element_by_xpath(
'.//a[contains(@href,"album?id") or contains(@href, "song?id")]').get_attribute("href")
_, _id = i.split("?id=")
if "song?id" in i:
song_set.add("https://music.163.com/#/song?id=" + _id)
else: # "album?id"
album_set.add("https://music.163.com/#/album?id=" + _id)
except NoSuchElementException:
pass
for album_url in album_set:
song_set.update(get_song_url_from_album_set(album_url))
return song_set
def save_song(url):
print("[{}] start".format(url), end=" ")
driver.get(url)
driver.switch_to.frame("contentFrame")
title = driver.find_element_by_xpath('//div[contains(@class, "tit")]').text
for word in shielding_words:
if word in title:
print("{} 因 {} 已忽略".format(title, word))
return
driver.find_element_by_xpath('//*[contains(text(), "收藏")]').click()
driver.find_element_by_xpath('//*[contains(text(), "{}")]'.format(song_dir)).click()
print(title)
driver = init_driver()
if not os.path.exists("tmp.txt"):
# 获取私信用户列表
private_detail_url_dict = get_private_detail()
print("private_detail_url_set len={}".format(len(private_detail_url_dict)))
# 获取歌曲id
song_url_set = set()
for private_detail_url, msg_num in private_detail_url_dict.items():
song_url_set.update(get_song_url_from_private_detail(private_detail_url, msg_num))
print("song_url_set len={}".format(len(song_url_set)))
with open("tmp.txt", "w", encoding="utf8") as f:
f.write("\n".join(song_url_set))
else:
with open("tmp.txt", "r", encoding="utf8") as f:
data = f.read()
song_url_set = data.split("\n") if data else []
# 保存歌曲
for song_url in song_url_set:
save_song(song_url)
os.remove("tmp.txt")
driver.close()

-----> [2018-07-16 17:22:42,041] [ERROR] [base.py<131>-base.run_job]: Job "auto_rollback.<locals>.wrapper (trigger: interval[0:30:00], next run at: 2018-07-16 17:52:42 CST)" raised an exception
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 250, in execute
self.errorhandler(self, exc, value)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 247, in execute
res = self._query(query)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 411, in _query
rowcount = self._do_query(q)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 374, in _do_query
db.query(q)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/connections.py", line 277, in query
_mysql.connection.query(self, query)
_mysql_exceptions.OperationalError: (2006, 'MySQL server has gone away')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/apscheduler/executors/base.py", line 125, in run_job
retval = job.func(*job.args, **job.kwargs)
File "/home/zza/eth_crawler/crawler_script/utils.py", line 28, in wrapper
raise err
File "/home/zza/eth_crawler/crawler_script/utils.py", line 24, in wrapper
return func(*args, **kwargs)
File "/home/zza/eth_crawler/crawler_script/token_tracker.py", line 240, in update
db_address = db.session.query(Token.contract_address).filter(None == Token.total_supply).all()
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 2773, in all
return list(self)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 2925, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 2948, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 186, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/usr/local/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 250, in execute
self.errorhandler(self, exc, value)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 247, in execute
res = self._query(query)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 411, in _query
rowcount = self._do_query(q)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/cursors.py", line 374, in _do_query
db.query(q)
File "/usr/local/lib/python3.6/site-packages/MySQLdb/connections.py", line 277, in query
_mysql.connection.query(self, query)
sqlalchemy.exc.OperationalError: (_mysql_exceptions.OperationalError) (2006, 'MySQL server has gone away') [SQL: 'SELECT token.contract_address AS token_contract_address \nFROM token \nWHERE token.total_supply IS NULL'] (Background on this error at: http://sqlalche.me/e/e3q8)
def auto_rollback(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as err:
db.session.rollback()
log.error(err)
raise err
return wrapper
app.config['SQLALCHEMY_POOL_SIZE'] = 128 # 线程池大小
app.config['SQLALCHEMY_POOL_TIMEOUT'] = 90 # 超时时间
app.config['SQLALCHEMY_POOL_RECYCLE'] = 3 # 空闲连接自动回收时间
app.config['SQLALCHEMY_MAX_OVERFLOW'] = 128 # 控制在连接池达到最大值后可以创建的连接数。
db.session.remove()
class nullpool_SQLAlchemy(SQLAlchemy):
def apply_driver_hacks(self, app, info, options):
super(nullpool_SQLAlchemy, self).apply_driver_hacks(app, info, options)
from sqlalchemy.pool import NullPool
options['poolclass'] = NullPool
del options['pool_size']
解决后又会出现
sqlalchemy.exc.InvalidRequestError: Can't reconnect until invalid transaction is rolled back
from xxx import SQLAlchemy
from xxx import app
db = SQLAlchemy(app)
最粗暴但是最有效的解决方式,这个问题困扰了将近 3 周,emmm

链接 来源:牛客网

在有幸体会过 996 的生活,早上 8.30 上班,中午休息两个小时,下午 6 点吃晚饭,晚饭后 7.30 上班到 9 点。
说是到九点,其实到了十点十一点还在公司,到宿舍洗漱后,差不多十二点了。
人是有硬性的娱乐时间(人必须有一定的自我放松时间,如果工作对自己来说不轻松的话,忘了那个地方说的了。),人是社会性动物,必须保证至少 4 个小时的社交 (《人类简史》)。996 的工作强度下,人的精神就很容易出问题。
再一个就是,在 996 的工作环境下,就只有周日了,周日是需要补觉的,那么一上午就没了,下午在看看生活必需品,那么连见朋友的时间都没了。
之前看过 广为人知 (che di feng sha) 的大项目 996.ICU,在国内环境确实感受到了严重的 内卷 和员工压榨。
确实也有很多讲述自身经验的方法,比如在 ICU 项目下:
因为家庭因素,我确实可以考虑出国发展,首选加拿大,加拿大最近有个 三年移民计划,对应的 加拿大 IT 人员计划 也有。发出来供大家参考。
加拿大,首先,考虑付费移民(如果有钱的话,五十万左右。去加拿大创业开店),相关信息,推荐搜索抖音号 [北美理李察德 (TopTalk)],非 IT 行业的方式都有,印象比较深的是去读加拿大的南翔技校,读个会计啥出来直接就业,逃离国内内卷环境。
IT 行业,我去加拿大的招聘网站看了看:
首先公司是支持完全远程上班的,对应的工资 19k-34k (参考 知乎,可能我搜索方式不对),然后英语其实要求很高。
对我而言,首要考虑把英语过了,雅思没到 7 分,其实大部分做不了。

为啥说大部分做不了呢,因为还有一条路,在中国找一家外企,比如 ThoughtWorks(思特沃克),做半年到一年后申请外调,到国外后,适应环境后在考虑把工作签证换成绿卡。
这种模式还有好处,在国内享受国外的制度,没有加班,年假长,福利好,被开除有很多很多的补偿。
简单搜索了一下,领英是最好的方式。参考 。其次是 glassdoor。
如果几种方式都不行,那么我想到的比较好的方式是,首先在国内支持远程上班的公司工作,然后在通过雅思去国外,这样在接触新环境的时候,不会因为脱产而产生过多的困难。
首先,远程上班的劣势要列一下(来自朋友分享):
与常规招聘渠道不太一样,这里有一下渠道共大家选择:

在疫情下,远程办公和远程面试得到了最大限度的发展。国内外很多公司已经完全支持远程办公,比如 Facebook,有点是 IT 终于摆脱地理限制,对国内而言,慢慢房价不再是一个问题,低成本同时意味着更大的竞争和更低的成本,国内首先考虑同样 10k 的价格,为何不雇一个已经会老家的 n 年经验的老员工,而在深圳雇佣一个 10k 的实习生,更大一点是,10k 的价格可以在印度雇佣三个程序猿。..
再这么一个内卷时代,先跳出来的人,就是第一个占位子的人。
换个地方呆确实是个考虑的方式,但是同时要考虑亲人朋友的感受,也同时要考虑会不会是从这座城进入了另一座城。