单独的 python 脚本文件使用 django 自带的 model

django1.9.5&python3.4.4

文件结构



在一个爬虫脚本中将爬取的数据通过django自带的model保存到数据库

修改的文件(其余pycharm新建Django项目生成,未修改):

# testapp/models.py
from django.db import models

class Problem(models.Model):
title = models.CharField(max_length=100, default="")
author = models.CharField(max_length=100, default="")
def __str__(self):
return self.title
pass

虚拟欢迎需要安装django

# push_view
import os
import sys
import django

pathname = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, pathname)
sys.path.insert(0, os.path.abspath(os.path.join(pathname, '..')))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xad_unit.settings")

django.setup()

from datetime import datetime
import time
from pv_uv.views import exec_sql
from pv_uv.models import Cpd

class Sql_Tool(object):

def zy_sql(self, date):
#sql = 'select ngx_date, path, event_type, count(distinct request_id) as num from xad_app_log where path = "/xad_tracking" and event_type in ("xad_dl_start", "xad_dl_success", "xad_install_start", "xad_install_success", "xad_click_tracking", "xad_impression") and ngx_date="{}"and ad_system="{}" group by ngx_date, path, event_type order by ngx_date, path, event_type limit 1000'.format(
#     date, 'liyanmobi')

sql = 'select ngx_date, path, event_type, count(distinct request_id) as num from xad_app_log where path = "/xad_tracking" and event_type in ("xad_dl_start", "xad_dl_success", "xad_install_start", "xad_install_success", "xad_click_tracking", "xad_impression") and ngx_date>="{}" and ad_system="{}" and req_from="1" group by ngx_date, path, event_type order by ngx_date, path, event_type limit 1000'.format(
date, 'liyanmobi')

l_list = exec_sql(sql)
return l_list

def strint_toDate(self, string):
return datetime.strptime(string, '%Y%m%d')

if __name__ == '__main__':

sql_tool = Sql_Tool()
current_time = time.strftime('%Y%m%d', time.localtime(time.time()))
#current_time = '20180726'
l_list = sql_tool.zy_sql(current_time)

print(l_list)
#l_list = [{'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_click_tracking', 'num': 302}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_dl_start', 'num': 182}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_dl_success', 'num': 101}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_impression', 'num': 301}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_install_start', 'num': 101}, {'ngx_date': '20180726', 'path': '/xad_tracking', 'event_type': 'xad_install_success', 'num': 42}]
l2 = l_list
l_list = list()
tmp_set = set()

for i_dict in l2:
tmp_dict = dict()
date1 = i_dict.get('ngx_date')
if date1 not in tmp_set:
tmp_set.add(date1)
else:
continue

for i_i_dict in l2:
date2 = i_i_dict.get('ngx_date')

if date1 == date2:
tmp_dict['ngx_date'] = i_i_dict.get('ngx_date')
tmp_dict[str(i_i_dict.get('event_type'))] = i_i_dict.get('num')
l_list.append(tmp_dict)
tmp_dict = dict()

print(l_list)
for i_dict in l_list:
cpd = Cpd()
for key, value in i_dict.items():

if key == 'ngx_date':
a = sql_tool.strint_toDate(i_dict.get('ngx_date'))
cpd.ngx_date = sql_tool.strint_toDate(i_dict.get('ngx_date'))
if key == 'xad_click_tracking':
cpd.xad_click_tracking = key
cpd.xad_click_tracking_nums = value
if key == 'xad_dl_start':
cpd.xad_dl_start = key
cpd.xad_dl_start_nums = value
if key == 'xad_dl_success':
cpd.xad_dl_success = key
cpd.xad_dl_success_nums = value
if key == 'xad_impression':
cpd.xad_impression = key
cpd.xad_impression_nums = value
if key == 'xad_install_start':
cpd.xad_install_start = key
cpd.xad_install_start_nums = value
if key == 'xad_install_success':
cpd.xad_install_success = key
cpd.xad_install_success_nums = value
cpd.save()