Skip to main content

New Blog

代码人生：编织技术与生活的博客之旅

需求​

  • 插入数据
  • 在默认排序中 数据显示在中间而不是最后

实验思路：

  • 按正常顺序插入 x 条数据
  • 拿第一条数据 加字段 id 不变 删除后加入

代码：

#!/usr/bin/python3    
# encoding: utf-8     
# @Time    : 2019/1/3 11:37    
# @Author  : zza    
# @Email   : 740713651@qq.com    
import datetime    
import time    
from pprint import pprint    
    
import pymongo    
from mongomock import ObjectId    
from tqdm import tqdm    
    
mongo_url = "mongodb://127.0.0.1:27017", 'sort_db'    
mongo_db = pymongo.MongoClient(mongo_url[0])[mongo_url[1]]    
    
def made_data():    
    mongo_db['demo'].drop()    
    print("开始造数据")    
    for i in tqdm(range(10)):    
        time.sleep(1)    
        mongo_db['demo'].insert({"timestamp": datetime.datetime.now()})    
    
def set_data():    
    # 之前想造个数据放进去  保存一下 如何生成 _id    
    # http://api.mongodb.com/python/current/api/bson/objectid.html    
    pprint(list(mongo_db['demo'].find()))    
    gen_time = datetime.datetime(2019, 1, 1, 14, 12, 26)    
    dummy_id = ObjectId.from_datetime(gen_time)    
    dummy_id = str(dummy_id)[:8] + "5c2da85ffc904a3c84335788"[8:]    
    dummy_id = ObjectId(dummy_id)    
    print(dummy_id)    
    result = mongo_db['demo'].insert({"_id": dummy_id, "info": "id made by python", "timestamp": gen_time})    
    print(result)    
    pprint(list(mongo_db['demo'].find()))    
    pprint(list(mongo_db['demo'].find({}).sort([("_id", -1)])))    
    
def re_insert_first():    
    a = mongo_db['demo'].find()[0]    
    mongo_db['demo'].delete_one({"_id": a["_id"]})    
    a.update({"info": "change by py"})    
    mongo_db['demo'].insert(a)    
    pprint(list(mongo_db['demo'].find()))    
    
made_data()    
# set_data()    
re_insert_first()

发现还是不行

后期的解决方案​

  • 在数据量不大的情况下
  • 删除后面的数据
  • 然后顺序插入
编程One min read

基本教程

  1. 腾讯企业邮箱转发到outlook邮箱 image.png image.png

  2. 将你的转发邮箱设置到安全邮件列表 image.png

  3. 设置普通JIRA规则围观类邮件归入普通文件夹 image.png

  4. 设置@我的JIRA规则 image.png

  5. 标记到微软待办(Microsoft To Do) image.png

  6. 下载微软待办(Microsoft To Do)客户端 略

  7. 使用你的outlook账号登录微软待办，查看待办事项 image.png

···

高端教程

  • windows下载outlook邮箱客户端
  • 设置规则如下 image.png 如此 只要你的客户端开着 提及到你的邮件就会自动出现在待办事项中

疑难杂症

  • 如果你的邮箱名为全数字无中文类型 则会出现邮件不出现在微软待办的问题 需要更换邮件名 包含英文才行

随笔One min read

pip 命令

运行以下命令就好了

pip config set global.index-url http://pypi.douban.com/simple    
pip config set global.trusted-host pypi.douban.com    
pip config set global.disable-pip-version-check true
  • 第三条用于关闭版本检查

部分操作系统可能有写入的位置不对的情况 于是乎有了下列脚本 前面配置已生效的同学可以不用管了

网上教程基本上都说 pip 的配置文件在用户目录下 %HOMEPATH%/.pip/pip 但是实际上在在 pip config list读的不知道是那个文件夹的文件 使用pip config set写入到%HOMEPATH%\AppData\Roaming\pip\pip.ini 用 --global --user 等指令 写入的地方也不同 有待研究

操作代码

#!/usr/bin/python3    
# encoding: utf-8     
# @Time    : 2020/4/13 14:12    
# @author  : zza    
# @Email   : 740713651@qq.com    
# @File    : pip_douban_source.py    
# https://python3-cookbook.readthedocs.io/zh_CN/latest/c13/p10_read_configuration_files.html    
# https://www.jianshu.com/p/0cdd647bcc3e    
"""    
python -c "import requests;res = requests.get('http://cdn.ricequant.com/rqpro/pip_douban_source_v2.py');exec(res.text)"    
"""    
    
import os    
import sys    
    
from configparser import ConfigParser    
    
conf_dir = os.path.join(os.path.expanduser("~"), ".pip")    
os.makedirs(conf_dir, exist_ok=True)    
WINDOWS = (sys.platform.startswith("win") or (sys.platform == 'cli' and os.name == 'nt'))    
CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf'    
conf_path = os.path.join(conf_dir, CONFIG_BASENAME)    
    
cfg = ConfigParser()    
cfg.read(conf_path, encoding="utf8")    
    
if not cfg.has_section('global'):    
    cfg.add_section('global')    
    
cfg.set('global', 'index-url', 'http://pypi.douban.com/simple')    
cfg.set('global', 'trusted-host', 'pypi.douban.com')    
cfg.set('global', 'timeout', "60")    
cfg.set('global', 'disable-pip-version-check', "true") #关闭版本检查    
    
if not WINDOWS:    
    if not cfg.has_section('install'):    
        cfg.add_section('install')    
    cfg.set("install", "use-mirrors", "true")    
    cfg.set("install", "mirrors", "https://pypi.douban.com/simple/")    
    cfg.set("install", "trusted-host", "pypi.douban.com")    
    
with open(conf_path, "w", encoding="utf8:") as f:    
    cfg.write(f)    
    
print("save to {}".format(conf_path))

一行命令跑起来

  • 这个文件我放公司 cdn 上了
  • 需要用到 python exec 方法 如下：
python -c "import requests;res = requests.get('http://cdn.ricequant.com/rqpro/pip_douban_source_v2.py');exec(res.text)"

Python2 min read

  • 按 ctrl+V 不能贴贴，ctrl+C 能复制
  • 选中文字后按 Backspace 不是删除，而是选定行往后继续+1
  • 原因：Pycharm 启动了 Vim 编辑模式
  • 解决方式：Tools -Vim Emulator 关闭就行了
  • 字典中，用变量名。get(key1) 和 变量名 [key1] 的区别
  • 变量名。get(key1) 如果没有给 0 值
  • 变量名 [key1] 没有则抛出 KeyError 异常
PythonOne min read

prometheus 监控 ，用gunicorn启动时。多进程内存不互通导致数据有问题。

参考​

gunicorn启动方案​

  • 安装prometheus_client pip install prometheus_client

  • 复制这个文件到你的项目中 vim monitoring.py

#!/usr/bin/python3    
# encoding: utf-8    
# @Time    : 2019/8/2 16:29    
# @author  : zza    
# @Email   : 740713651@qq.com    
# @File    : monitoring.py    
"""    
    FROM https://github.com/ITISFoundation/osparc-simcore/blob/3e80ce451352c906f2876113dbb6ae33e8574be1/packages/service-library/src/servicelib/monitoring.py    
    &&  https://github.com/ITISFoundation/osparc-simcore/blob/3e80ce451352c906f2876113dbb6ae33e8574be1/packages/service-library/src/servicelib/monitoring.py    
"""    
import time    
    
from flask import request, current_app, Response    
from prometheus_client import Counter, Histogram    
from prometheus_client import multiprocess    
from prometheus_client import generate_latest, CollectorRegistry, CONTENT_TYPE_LATEST, Gauge    
    
# Example gauge.    
IN_PROGRESS = Gauge("inprogress_requests", "help", multiprocess_mode='livesum')    
    
    
# Expose metrics.    
@IN_PROGRESS.track_inprogress()    
def app(environ, start_response):    
    registry = CollectorRegistry()    
    multiprocess.MultiProcessCollector(registry)    
    data = generate_latest(registry)    
    status = '200 OK'    
    response_headers = [    
        ('Content-type', CONTENT_TYPE_LATEST),    
        ('Content-Length', str(len(data)))    
    ]    
    start_response(status, response_headers)    
    return iter([data])    
    
    
def setup_monitoring(app, app_name=None):    
    if app_name is None:    
        app_name = app.name    
    
    def start_timer():    
        request.start_time = time.time()    
        current_app.extensions["prometheus"]['REQUEST_IN_PROGRESS'].labels(    
            app_name, request.endpoint, request.method).inc()    
    
    def record_request_data(response):    
        resp_time = time.time() - request.start_time    
        endpoint = request.endpoint    
        ext_prometheus = current_app.extensions["prometheus"]    
        ext_prometheus['REQUEST_LATENCY'].labels(app_name, endpoint).observe(resp_time)    
        ext_prometheus['REQUEST_IN_PROGRESS'].labels(app_name, endpoint, request.method).dec()    
        ext_prometheus['REQUEST_COUNT'].labels(app_name, request.method, endpoint, response.status).inc()    
        return response    
    
    app.before_request(start_timer)    
    app.after_request(record_request_data)    
    
    extensions_prometheus = dict()    
    extensions_prometheus['app_name'] = app_name    
    extensions_prometheus['REQUEST_COUNT'] = Counter(    
        'http_requests_total', 'Total Request Count',    
        ['app_name', 'method', 'endpoint', 'http_status']    
    )    
    
    # Latency of a request in seconds    
    extensions_prometheus['REQUEST_LATENCY'] = Histogram(    
        'http_request_latency_seconds', 'Request latency',    
        ['app_name', 'endpoint']    
    )    
    
    extensions_prometheus['REQUEST_IN_PROGRESS'] = Gauge(    
        'http_requests_in_progress_total', 'Requests in progress',    
        ['app_name', 'endpoint', 'method']    
    )    
    
    app.extensions["prometheus"] = extensions_prometheus    
    
    @app.route("/metrics")    
    def metrics():    
        registry = CollectorRegistry()    
        multiprocess.MultiProcessCollector(registry)    
        data = generate_latest(registry)    
        return Response(data, mimetype=CONTENT_TYPE_LATEST)
  • 在你代码中 导入文件并初始化
# from flask import Flask    
# app = Flask(__name__)    
from persistd.monitoring import setup_monitoring    
setup_monitoring(app, "app_name")
  • 设置Gunicom配置文件 vim gunicorn.conf.py
from prometheus_client import multiprocess    
def child_exit(server, worker):    
    multiprocess.mark_process_dead(worker.pid)

  • 启动Gunicom时 增加参数指向配置文件 -c gunicorn.conf.py

  • 设置环境变量：需要一个临时文件夹,且环境变量prometheus_multiproc_dir指向该文件夹(注意启动用户读写权限), 该文件夹用于存放prometheus数据。

rm -rf multiproc-tmp    
mkdir multiproc-tmp    
export prometheus_multiproc_dir=multiproc-tmp    
gunicorn -c gunicorn_conf.py -w 4 yourapp:app

附一个 asyncio 的 monitoring.py

#!/usr/bin/python3    
# encoding: utf-8    
# @Time    : 2019/9/5 16:36    
# @author  : zza    
# @File    : monitoring.py    
"""    
FROM:    
    https://github.com/cloud-cds/cds-stack/blob/4243cd9b2e878f16a251d05afb2d202d71e41dce/api/monitoring.py    
    https://github.com/DD-DeCaF/gene-to-reactions/blob/3af42110433edf8495810e6a95a516368464e179/src/gene_to_reactions/app.py    
    
    setup_monitoring(app, "app_name")    
"""    
import time    
import asyncio    
from aiohttp import web    
from prometheus_client import multiprocess, generate_latest    
from prometheus_client import CONTENT_TYPE_LATEST, CollectorRegistry, Histogram, Counter, Gauge    
    
    
def prom_middleware(app_name):    
    @asyncio.coroutine    
    def factory(app, handler):    
        @asyncio.coroutine    
        def middleware_handler(request):    
            try:    
                request['start_time'] = time.time()    
                request.app['REQUEST_IN_PROGRESS'].labels(    
                    app_name, request.path, request.method).inc()    
                response = yield from handler(request)    
                resp_time = time.time() - request['start_time']    
                request.app['REQUEST_LATENCY'].labels(app_name, request.path).observe(resp_time)    
                request.app['REQUEST_IN_PROGRESS'].labels(app_name, request.path, request.method).dec()    
                request.app['REQUEST_COUNT'].labels(    
                    app_name, request.method, request.path, response.status).inc()    
                return response    
            except Exception as ex:    
                raise    
    
        return middleware_handler    
    
    return factory    
    
    
async def metrics(request):    
    resp = web.Response(body=generate_latest(multiprocess.MultiProcessCollector(CollectorRegistry())))    
    resp.content_type = CONTENT_TYPE_LATEST    
    return resp    
    
    
def setup_monitoring(app, app_name):    
    app['REQUEST_COUNT'] = Counter(    
        'requests_total', 'Total Request Count',    
        ['app_name', 'method', 'endpoint', 'http_status']    
    )    
    app['REQUEST_LATENCY'] = Histogram(    
        'request_latency_seconds', 'Request latency',    
        ['app_name', 'endpoint']    
    )    
    
    app['REQUEST_IN_PROGRESS'] = Gauge(    
        'requests_in_progress_total', 'Requests in progress',    
        ['app_name', 'endpoint', 'method']    
    )    
    
    app.middlewares.insert(0, prom_middleware(app_name))    
    app.router.add_get("/metrics", metrics)
Python2 min read

  • 跑程序的时候发现 _id 相同冲突
  • 程序中 mongo_id 的生成有 timestamp 决定
  • 于是测试：
import time    
for i in range(20):    
    print(time.time())    
    
a = time.time()    
for i in range(10**8):    
    time.time()    
print(time.time() - a)

结果：

1546937536.8363261    
1546937536.8363261    
1546937536.8363261    
1546937536.8363261    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
1546937536.8372946    
9.680140972137451

  • emmm 经过与同事的机器的对比 短时间内

  • Windows 上时间戳是相同

  • mac 不同

  • 似乎是系统 bug

  • 各位要用时间戳做 id 的同学注意下

  • 注意

  • 1546937536.8363261

  • 1546937536.8372946

  • 时间不是连续的

  • time 的精度不同

PythonOne min read

[ ] 表示博主没有学过的 [x] 表示博主已经学过的

6 大设计原则

介绍的比较详细 静下心看

[x] 设计模式六大原则

23 种设计模式

每种设计模式 先看 1 再看 2

图解 23 种设计模式 无代码

1《大话设计模式》Python 版代码实现

23 设计模式的代码实现

2 二十三种设计模式及其 python 实现

数学

[ ] 概率论 [ ] 数理统计 [x] 高等数学 [x] 线性代数 [x] 离散数学 [x] 复变函数与积分变换

编程方面

[ ] 数据挖掘 [ ] 人工智能 [ ] 分布式计算 [x] 数据结构 [x] 算法分析与设计 [x] 软件工程 [x] 面向对象系统分析与设计 [x] 软件项目管理 [x] 软件测试 [x] 计算机网络 [x] 编译原理 [x] 操作系统 [x] 信息安全技术概论 [x] 数据库 Oracle [x] 数据库 SQLServer

其他课

[x] 就业指导 [x] 计算机操作及维护

硬件方面略

编程One min read

要求​

  • 一亿数据 10*8
  • 测试用 5w 数据
  • 预测时间为 结果时间* 2000

设计思路​

  • 程序执行 20 遍求平均值
  • 结束时间 - 开始时间
  • 不同 python 引擎
  • 不同数据量 然后 commit 提交 响应速度

数据库连接工具​

  • MySQL-Python
  • pymysql
  • [X ] MySQL-Connector

代码​

    #!/usr/bin/python3    
# encoding: utf-8     
# @Time    : 2018/7/14 0014 16:12    
# @author  : zza    
# @Email   : 740713651@qq.com    
import time    
    
from flask import Flask    
from flask_sqlalchemy import SQLAlchemy    
    
db = SQLAlchemy()    
    
class Student(db.Model):    
    __tablename__ = "stu"    
    id = db.Column(db.Integer, primary_key=True, autoincrement=True, nullable=True)    
    name_ = db.Column(db.String(127))    
    age = db.Column(db.Integer)    
    class_num = db.Column(db.Integer)    
    
def init(param):    
    class sqlalchemy(SQLAlchemy):    
    
        def __del__(self):    
            print("数据库关闭")    
            db.session.close_all()    
    
    app = Flask(__name__)    
    app.config['SQLALCHEMY_DATABASE_URI'] = param + "?charset=utf8&autocommit=False"    
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True    
    app.config['SQLALCHEMY_POOL_SIZE'] = 128    
    app.config['SQLALCHEMY_POOL_TIMEOUT'] = 60    
    app.config['SQLALCHEMY_POOL_RECYCLE'] = 30    
    app.config['SQLALCHEMY_MAX_OVERFLOW'] = 128    
    # app.config['SQLALCHEMY_ECHO'] = True    
    global db    
    db = sqlalchemy(app)    
    
def finish():    
    db.session.query(Student).delete()    
    db.session.commit()    
    
def time_me(fn):    
    def _wrapper(*args, **kwargs):    
        average = 0    
        i1 = 30    
        seconds = 0    
        for i in range(i1):    
            start = time.time()    
            fn(*args, **kwargs)    
            seconds = time.time() - start    
            average += seconds    
        finish()    
        print(u"{func}函数写入耗时{sec}秒".format(func=fn.__name__, sec=seconds))    
        # print(u"{func}函数每{count}条数数据写入耗时{sec}秒".format(func=fn.__name__, count=args[0], sec=seconds))    
        # finish()    
        return seconds, args    
    
    return _wrapper    
    
@time_me    
def insert_many():    
    # 插入诗句    
    all = 5 * 10 ** 4    
    inner = 1000    
    out = int(all / inner)    
    for i in range(out):    
        for c in range(inner):    
            db.session.add(Student(name_='test mysql insert', age=30, class_num=30))    
        db.session.commit()    
    
######    
@time_me    
def insert_many_by_sql():    
    all = 5 * 10 ** 4    
    inner = 1000    
    out = int(all / inner)    
    with db.session.connection() as con:    
        for i in range(out):    
            for c in range(inner):    
                con.execute(    
                    "INSERT INTO stu ( id ,name_, age, class_num) VALUES (null ,{}, {},{})".format(    
                        "'test2mysql3insert'",    
                        30, 30))    
        db.session.commit()    
    
def main2():    
    """测试 sql 语句与 orm 框架 谁快  包括数据组装"""    
    init("mysql+pymysql://root:root@192.168.14.147:3306/efficiency_test")    
    print("orm 框架插入数据")    
    # iinsert_many 函数每 500 条数数据写入耗时 19.671629905700684 秒    
    insert_many()    
    print("sql 语句插入数据")    
    # insert_many_by_sql 函数每 500 条数数据写入耗时 17.977628707885742 秒    
    insert_many_by_sql()    
    pass    
    
def main():    
    print('测试开始')    
    # insert_many 函数写入耗时 168.07286262512207 秒    
    init("mysql+mysqlconnector://root:root@192.168.14.147:3306/efficiency_test")    
    insert_many()    
    
    # insert_many 函数写入耗时 64.85304117202759 秒    
    init("mysql://root:root@192.168.14.147:3306/efficiency_test")  # 默认使用 MySQLdb    
    insert_many()    
    
    # insert_many 函数写入耗时 64.692676067352295 秒    
    init("mysql+pymysql://root:root@192.168.14.147:3306/efficiency_test")    
    insert_many()    
    
    # insert_many 函数写入耗时 66.991496086120605 秒    
    init("mysql+mysqldb://root:root@192.168.14.147:3306/efficiency_test")    
    insert_many()    
    
if __name__ == '__main__':    
    main()    
    main2()
Python2 min read