w

hhh/GATest.py

@@ -1,12 +1,33 @@
 """Hello Analytics Reporting API V4."""
-
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
 from apiclient.discovery import build
 from oauth2client.service_account import ServiceAccountCredentials
+import dataset
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+db.query('delete from hhh_weekly_report')
 
+table=db['hhh_weekly_report']
 
 SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
-KEY_FILE_LOCATION = ''
-VIEW_ID = '<REPLACE_WITH_VIEW_ID>'
+KEY_FILE_LOCATION = 'c:\\keys\\choozmo-ga-beee24b7a4c1.json'
+VIEW_ID = '188916214'
 
 
 def initialize_analyticsreporting():
@@ -24,7 +45,7 @@ def initialize_analyticsreporting():
   return analytics
 
 
-def get_report(analytics):
+def get_report(analytics,body):
   """Queries the Analytics Reporting API V4.
 
   Args:
@@ -34,13 +55,18 @@ def get_report(analytics):
   """
   return analytics.reports().batchGet(
       body={
-        'reportRequests': [
-        {
-          'viewId': VIEW_ID,
-          'dateRanges': [{'startDate': '7daysAgo', 'endDate': 'today'}],
-          'metrics': [{'expression': 'ga:sessions'}],
-          'dimensions': [{'name': 'ga:country'}]
-        }]
+        'reportRequests':body
+        # [
+        #{
+#          'viewId': VIEW_ID,
+#          'dateRanges': [{'startDate': '14daysAgo', 'endDate': 'today'}],
+#          'dateRanges': [{'startDate': '2021-05-30', 'endDate': '2021-06-05'}],
+
+#          'metrics': [{'expression': 'ga:users'}],
+#          'dimensions': [{'name': 'ga:sourceMedium'}]
+#          'dimensions': [{'name': 'ga:date'},{'name': 'ga:sourceMedium'}]
+
+#        }]
       }
   ).execute()
 
@@ -51,6 +77,7 @@ def print_response(response):
   Args:
     response: An Analytics Reporting API V4 response.
   """
+  result=[]
   for report in response.get('reports', []):
     columnHeader = report.get('columnHeader', {})
     dimensionHeaders = columnHeader.get('dimensions', [])
@@ -59,20 +86,53 @@ def print_response(response):
     for row in report.get('data', {}).get('rows', []):
       dimensions = row.get('dimensions', [])
       dateRangeValues = row.get('metrics', [])
+      ga_dict={}
 
       for header, dimension in zip(dimensionHeaders, dimensions):
         print(header + ': ', dimension)
-
+        ga_dict[header]=dimension
+      
       for i, values in enumerate(dateRangeValues):
         print('Date range:', str(i))
         for metricHeader, value in zip(metricHeaders, values.get('values')):
-          print(metricHeader.get('name') + ':', value)
-
+          ga_dict[metricHeader.get('name')]=value
+#          print(metricHeader.get('name') + ':', value)
+        result.append(ga_dict)
+    return result
+#        print(ga_dict)
 
 def main():
   analytics = initialize_analyticsreporting()
-  response = get_report(analytics)
-  print_response(response)
+#(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)
+
+  body=[{ 'viewId': VIEW_ID,
+  'dateRanges': [{'startDate': '2021-05-24', 'endDate': '2021-06-10'}],
+#  'filtersExpression': 'ga:sourceMedium=~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
+  'filtersExpression': 'ga:sourceMedium=~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
+  'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
+  'dimensions': [{'name': 'ga:date'},{'name': 'ga:isoWeek'}]
+  }]
+
+  response = get_report(analytics,body)
+  ga_dict=print_response(response)
+  for elmt in ga_dict:
+      elmt['category']='social'
+      table.insert(elmt)
+
+  body=[{ 'viewId': VIEW_ID,
+  'dateRanges': [{'startDate': '2021-05-24', 'endDate': '2021-06-10'}],
+#  'filtersExpression': 'ga:sourceMedium=~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
+  'filtersExpression': 'ga:sourceMedium!~(FB_|facebook|IG_|LINE_|LINEMP_|qsear.ch)',
+  'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
+  'dimensions': [{'name': 'ga:date'},{'name': 'ga:isoWeek'}]
+  }]
+
+  response = get_report(analytics,body)
+  ga_dict=print_response(response)
+  for elmt in ga_dict:
+      elmt['category']='organic'
+      table.insert(elmt)
+
 
 if __name__ == '__main__':
   main()

hhh/GATest2.py

@@ -0,0 +1,136 @@
+"""Hello Analytics Reporting API V4."""
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+from apiclient.discovery import build
+from oauth2client.service_account import ServiceAccountCredentials
+import dataset
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+db.query('delete from hhh_weekly_affinity')
+
+table=db['hhh_weekly_affinity']
+
+SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
+KEY_FILE_LOCATION = 'c:\\keys\\choozmo-ga-beee24b7a4c1.json'
+VIEW_ID = '188916214'
+
+
+def initialize_analyticsreporting():
+  """Initializes an Analytics Reporting API V4 service object.
+
+  Returns:
+    An authorized Analytics Reporting API V4 service object.
+  """
+  credentials = ServiceAccountCredentials.from_json_keyfile_name(
+      KEY_FILE_LOCATION, SCOPES)
+
+  # Build the service object.
+  analytics = build('analyticsreporting', 'v4', credentials=credentials)
+
+  return analytics
+
+
+def get_report(analytics,body):
+  """Queries the Analytics Reporting API V4.
+
+  Args:
+    analytics: An authorized Analytics Reporting API V4 service object.
+  Returns:
+    The Analytics Reporting API V4 response.
+  """
+  return analytics.reports().batchGet(
+      body={
+        'reportRequests':body
+        # [
+        #{
+#          'viewId': VIEW_ID,
+#          'dateRanges': [{'startDate': '14daysAgo', 'endDate': 'today'}],
+#          'dateRanges': [{'startDate': '2021-05-30', 'endDate': '2021-06-05'}],
+
+#          'metrics': [{'expression': 'ga:users'}],
+#          'dimensions': [{'name': 'ga:sourceMedium'}]
+#          'dimensions': [{'name': 'ga:date'},{'name': 'ga:sourceMedium'}]
+
+#        }]
+      }
+  ).execute()
+
+
+def print_response(response):
+  """Parses and prints the Analytics Reporting API V4 response.
+
+  Args:
+    response: An Analytics Reporting API V4 response.
+  """
+  result=[]
+  for report in response.get('reports', []):
+    columnHeader = report.get('columnHeader', {})
+    dimensionHeaders = columnHeader.get('dimensions', [])
+    metricHeaders = columnHeader.get('metricHeader', {}).get('metricHeaderEntries', [])
+
+    for row in report.get('data', {}).get('rows', []):
+      dimensions = row.get('dimensions', [])
+      dateRangeValues = row.get('metrics', [])
+      ga_dict={}
+
+      for header, dimension in zip(dimensionHeaders, dimensions):
+        print(header + ': ', dimension)
+        ga_dict[header]=dimension
+      
+      for i, values in enumerate(dateRangeValues):
+        print('Date range:', str(i))
+        for metricHeader, value in zip(metricHeaders, values.get('values')):
+          ga_dict[metricHeader.get('name')]=value
+#          print(metricHeader.get('name') + ':', value)
+        result.append(ga_dict)
+    return result
+#        print(ga_dict)
+
+def main():
+  analytics = initialize_analyticsreporting()
+
+  body=[{ 'viewId': VIEW_ID,
+#  'dateRanges': [{'startDate': '2021-05-30', 'endDate': '2021-06-05'}],
+  'dateRanges': [{'startDate': '2021-05-28', 'endDate': '2021-06-10'}],
+  'metrics': [{'expression': 'ga:users'}],
+
+#  'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
+#  'metrics': [{'expression': 'ga:users'},{'expression': 'ga:newusers'},{'expression': 'ga:sessions'},{'expression': 'ga:pageviews'},{'expression': 'ga:bounceRate'},{'expression': 'ga:pageviewsPerSession'}],
+#  'dimensions': [{'name': 'ga:date'},{'name': 'ga:sourceMedium'}]
+#  'dimensions': [{'name': 'ga:date'}]
+  'dimensions': [{'name': 'ga:interestAffinityCategory'},{'name': 'ga:isoWeek'},]
+
+  }]
+
+  response = get_report(analytics,body)
+
+
+  ga_dict=print_response(response)
+#  print(ga_dict)
+  for elmt in ga_dict:
+    elmt['ga:users']=int(elmt['ga:users'])
+    if elmt['ga:users'] > 5000:
+      print(elmt['ga:interestAffinityCategory'])
+      table.insert(elmt)
+
+#        print(elmt['ga:sourceMedium'])
+
+
+if __name__ == '__main__':
+  main()
+

hhh/GA_DB_KW_to_Sheep.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('/var/keys/spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+def do_jobs():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    cursor=db.query("SELECT k1.query,k1.clicks,k1.impressions,k1.ctr,(k2.clicks-k1.clicks)/k1.clicks*100 as growth FROM hhh.hhh_weekly_keywords k1, hhh_weekly_keywords k2 where k1.weeknum='22' and k2.weeknum='23' and k1.query=k2.query order by (k2.clicks-k1.clicks)/k1.clicks*100 desc;")
+    df = pd.DataFrame(columns=('query','clicks','impressions','ctr','growth'))
+
+    idx=0
+
+    for c in cursor:
+        df.loc[idx]=[c['query'],c['clicks'],c['impressions'],c['ctr'],c['growth']]
+    #    df.loc[idx]=['okok',333]
+        idx+=1
+    save_sheet(df,'WeeklyReport','keywords')
+
+t = threading.Thread(target = do_jobs)
+t.start()
+t.join()
+

hhh/GA_DB_to_Sheep.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python3
+import sys
+import codecs
+import traceback
+import requests
+import re
+import pandas as pd
+import random
+import urllib
+import dataset
+import json
+import gspread
+import datetime
+from gspread_pandas import Spread, Client
+from oauth2client.service_account import ServiceAccountCredentials
+import os
+import threading
+
+def save_sheet(df,filename,tabname,startpos='A1'):
+
+    scope = ['https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive']
+
+    credentials = ServiceAccountCredentials.from_json_keyfile_name('c:\\keys\\spread2.json', scope)
+#    credentials = ServiceAccountCredentials.from_json_keyfile_name('/var/keys/spread2.json', scope)
+
+    gc = gspread.authorize(credentials)
+    spread = Spread(filename,creds=credentials)
+
+    spread.df_to_sheet(df, index=False, sheet=tabname, start=startpos, replace=False)
+
+
+def do_jobs():
+    db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+    cursor=db.query('SELECT `ga:date`,`ga:users`,`ga:newusers`,`ga:sessions`,`ga:pageviews`,`ga:bounceRate`,`ga:pageviewsPerSession`,`ga:isoWeek`,category FROM hhh.hhh_weekly_report;')
+    df = pd.DataFrame(columns=('date','users','newusers','sessions','pageviews','bounceRate','pv_session','week','category'))
+
+    idx=0
+
+    for c in cursor:
+        df.loc[idx]=[c['ga:date'],c['ga:users'],c['ga:newusers'],c['ga:sessions'],c['ga:pageviews'],c['ga:bounceRate'],c['ga:pageviewsPerSession'],c['ga:isoWeek'],c['category']]
+    #    df.loc[idx]=['okok',333]
+        idx+=1
+    save_sheet(df,'WeeklyReport','rawdata')
+
+t = threading.Thread(target = do_jobs)
+t.start()
+t.join()
+

hhh/GA_Keywords.py

@@ -0,0 +1,58 @@
+import traceback
+import copy
+import operator
+import codecs
+import sys
+import os
+import searchconsole
+import dataset
+import datetime
+import time
+
+
+fname=os.path.abspath(__file__)
+elmts=fname.split(os.path.sep)
+path2=os.path.sep.join(elmts[0:-1])
+keysdir=path2+os.path.sep+'keys'+os.path.sep
+
+account = searchconsole.authenticate(client_config=keysdir+'client_secret.json',credentials=keysdir+'credentials.json')
+webproperty = account['https://hhh.com.tw/']
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/hhh?charset=utf8mb4')
+#db = dataset.connect('mysql://choozmo:pAssw0rd@127.0.0.1:3306/hhh?charset=utf8mb4')
+db.query('delete from hhh_weekly_keywords')
+table=db['hhh_weekly_keywords']
+records=[]
+
+def gen_report(weeknum,begindate,enddate):
+  #report=webproperty.query.range('2021-01-01', '2020-01-02').dimension('page','query').get()
+  report=webproperty.query.range(begindate, enddate).dimension('query').get()
+
+  ts = time.time()
+  sessionid=int(ts)
+
+  for r in report:
+  #    print(r)
+      query=r[0]
+      clicks=int(r[1])
+      impressions=r[2]
+      ctr=r[3]
+      position=r[4]
+      if clicks >=200:
+        records.append({'sessionid':sessionid,'query':query,'clicks':clicks,'impressions':int(impressions),'ctr':float(ctr),'position':float(position),'dt':datetime.datetime.now(),'weeknum':weeknum})
+
+
+gen_report('21','2021-05-24','2020-05-30')
+gen_report('22','2021-05-31','2020-06-06')
+gen_report('23','2021-06-07','2020-06-13')
+
+print('complete')
+
+db.begin()
+for r in records:
+    try:
+        table.insert(r)
+        db.commit()
+    except:
+        db.rollback()
+print('done')

hhh/gspace_fetch_ranks.py

@@ -57,8 +57,9 @@ def get_designer_statistics(designer_list):
 
 
 
-    for q in designer_list:
-
+    for tup in designer_list:
+        q=tup[0]
+        vip=tup[1]
         googleurl='https://www.google.com/search?q='+q
 
         jb.get(googleurl)
@@ -113,7 +114,7 @@ def get_designer_statistics(designer_list):
                     print("updsert")
                     print({'name':q,'updated':datetime.datetime.now()})
                     table.upsert({'name':q,'updated':datetime.datetime.now()},keys=['name'])
-                    m={'designer':q,'title':txt,'url':href,'hhh':hhh,'dt':datetime.datetime.now(),'searchome':searchhome,'c_100':com100}
+                    m={'designer':q,'title':txt,'url':href,'hhh':hhh,'dt':datetime.datetime.now(),'searchome':searchhome,'c_100':com100,'vip':vip}
                     masters.append(m)
                     table_summary.insert(m)
 
@@ -169,9 +170,11 @@ def get_designer_statistics(designer_list):
 
 
 qlist=[]
-cursor=db.query('select name from customer_list order by updated asc limit 20')
+#cursor=db.query('select name,vip from customer_list order by updated asc limit 20')
+cursor=db.query('select name,vip from customer_list where vip =  0 order by updated asc limit 20')
+
 for c in cursor:
-    qlist.append(c['name'])
+    qlist.append((c['name'],c['vip']))
 
 #get_designer_statistics([qlist[0]])
 get_designer_statistics(qlist)

tests/mtest.py

@@ -4,35 +4,38 @@ import os
 from email.mime.text import MIMEText
 from email.mime.image import MIMEImage
 from email.mime.multipart import MIMEMultipart
-
+from email.message import EmailMessage
+import codecs
 gmail_user = 'edm@choozmo.com'
 gmail_password='wqdsyqwvppmubitv'
 
-
-
 sent_from = gmail_user
-to = ['jeweiliang@gmail.com']
+to = ['jared@choozmo.com','nina.huang@choozmo.com','ana@choozmo.com','ming@choozmo.com','mike@choozmo.com','andy@choozmo.com','hana@choozmo.com','stacy@choozmo.com','wen@choozmo.com','yukyo@choozmo.com','fxp87257@gmail.com','noodlesloves@gmail.com']
 img_data=None
 with open('c:/tmp/coupon.png', 'rb') as f:
     img_data = f.read()
 
 msg = MIMEMultipart()
-msg['Subject'] = '幸福空間五萬裝修折價券'
+msg['Subject'] = '[2021-06-11] 趨勢日報 | ChoozMo '
 msg['From'] = 'edm@choozmo.com'
 msg['To'] = 'jeweiliang@gmail.com'
 
-text = MIMEText("感謝您填寫問卷，敬送您五萬元裝修折價券。")
-msg.attach(text)
-image = MIMEImage(img_data, name=os.path.basename("c:/tmp/coupon.png"))
-msg.attach(image)
+msgAlternative = MIMEMultipart('alternative')
+msg.attach(msgAlternative)
 
+fr=codecs.open('c:/tmp/final.html','r','utf-8')
+content=fr.read()
+fr.close()
+
+text = MIMEText(content,'html','utf-8')
+
+msgAlternative.attach(text)
 try:
     server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
     server.ehlo()
     server.login(gmail_user, gmail_password)
     server.sendmail(sent_from, to,  msg.as_string())
     server.close()
-
     print ('Email sent!')
 except:
     traceback.print_exc()

tests/testtmpl.py

@@ -0,0 +1,37 @@
+import jinja2
+import codecs
+import os
+import dataset
+
+db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/yodb?charset=utf8mb4')
+
+
+fname=os.path.abspath(__file__)
+elmts=fname.split(os.path.sep)
+path2=os.path.sep.join(elmts[0:-1])
+keysdir=path2+os.path.sep
+print(keysdir)
+
+
+templateLoader = jinja2.FileSystemLoader(searchpath=keysdir)
+templateEnv = jinja2.Environment(loader=templateLoader)
+TEMPLATE_FILE = 'test.tmpl'
+template = templateEnv.get_template(TEMPLATE_FILE)
+
+
+#SELECT distinct ts_word FROM trending_searches ORDER BY ts_date DESC limit 21;
+cursor=db.query('SELECT distinct ts_word FROM trending_searches ORDER BY ts_date DESC limit 21;')
+display={}
+idx=1
+for c in cursor:
+    display['a'+str(idx)]=c['ts_word']
+    idx+=1
+
+outputText = template.render(display=display) 
+
+fw=codecs.open('c:/tmp/final.html','w','utf-8')
+fw.write(outputText)
+fw.close()
+
+os.startfile('c:/tmp/final.html')
+