kw_tools/SEO/ranking_world.py

# import redis
import time
import traceback
# import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time
import os
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import dataset
from selenium.webdriver.common.keys import Keys
import json
import random
import time
import sys
import codecs
import pandas as pd
import random
import os
import time
import datetime
from selenium.webdriver.chrome.service import Service
import dataset
import pymysql
import undetected_chromedriver as uc


pymysql.install_as_MySQLdb()


import requests

driver = None

db = dataset.connect('mysql://choozmo:pAssw0rd@db.ptt.cx:3306/seo?charset=utf8mb4')

headers = {
    "Authorization": "Bearer " + "6SDULL1Ebklduc6TFxa97AFto5Sj21kyJ30CxiLiSoi",
    "Content-Type": "application/x-www-form-urlencoded"
}


def send_msg(kw):
    params = {"message": kw}
    r = requests.post("https://notify-api.line.me/api/notify", headers=headers, params=params)


def re_get_webdriver():
    global port
    global driver
    result = []
    if driver is not None:
        print('closing....')
        driver.quit()
        os.system('killall chrome')
        print('quit....')
        driver = None
    try:
        s = Service('C:\/Users\/s1301\/Downloads\/136\/chromedriver-win32\/chromedriver.exe')
        options = uc.ChromeOptions()
        options.add_argument("--window-size=200,100")  # 縮小視窗
        options.add_argument("--window-position=-32000,-32000")  # 移到螢幕外
        driver = uc.Chrome(options=options,service=s)
        driver.delete_all_cookies()
        return driver
    except:
        traceback.print_exc()
        driver = None
        return None


def scrolling(driver, pgnum):
    ub = driver.find_element("css selector",'body')
    for i in range(pgnum):
        ub.send_keys(Keys.PAGE_DOWN)
        if pgnum > 1:
            time.sleep(0.3)


def run_once(jsobj):
    table = db['google_rank']
    date = jsobj['date']
    print(jsobj)
    global driver

    i = 100
    if driver is None:
        time.sleep(8)
        re_get_webdriver()
    if driver is None:
        return
    try:
        kw = jsobj['kw']
        fname = jsobj['fname']
        url = jsobj['url']
        # if jsobj.get('domain') is None:
        #     exclude = jsobj['exclude']
        #     domain = None
        # else:
        #     domain = jsobj['domain']
        #     exclude = None
        city_map = {'chicago': ['42.04866173771369', '-87.68260072643513'],
                    'miami': ['25.764458843530548', '-80.19787522585152'],
                    'wc': ['38.96071674051165', '-77.03155367248387'],
                    'ny': ['40.76774141099703', '-73.98439238945637']}
        city = jsobj['fname']
        print(city)
        # Map_coordinates = dict({
        #     "latitude": float(city_map[f"{city}"][0]),
        #     "longitude": float(city_map[f"{city}"][1]),
        #     "accuracy": 100
        # })

        # 芝加哥、邁阿密、紐約、華盛頓
        # driver.execute_cdp_cmd("Emulation.setGeolocationOverride", Map_coordinates)
        # driver.get('https://www.google.com?num=100&lr=lang_en')


        driver.get(url)
        # print(driver.current_url)
        # elmt = driver.find_element(By.XPATH, "//input[@name='q']")
        # time.sleep(1)
        # elmt.send_keys(kw)
        #
        # elmt.send_keys(Keys.ENTER)
        time.sleep(3)
        scrolling(driver, 10)
        time.sleep(60)

        elmts = driver.find_elements(By.XPATH, "//div[@class='yuRUbf']//a")
        numresults = len(elmts)
        print('搜尋結果數量', numresults)
        time.sleep(20)
        driver.save_screenshot(r"C:\/Users\/s1301\/Downloads\/" +date +fname + ".png")
        if numresults == 0:
            send_msg('stop working...')
            sys.exit()
        datadict = {'搜尋詞': [], '結果標題': [], '結果網址': [], '結果名次': [], '結果說明': []}
        df = pd.DataFrame()

        idx = 1
        found = False
        test_lst = []
        txt_dict = {}
        for elmt in elmts:
            href = elmt.get_attribute('href')
            txt = elmt.text
            desc = None
            try:
                elmt2 = elmt.find_element(By.XPATH, "./../../..//div[@data-content-feature=1]")
                desc = elmt2.text
            except:
                desc = None
            #            print(desc)
            table.insert(
                {'title': elmt.text, 'url': href, 'keyword': kw, 'dt': datetime.datetime.now(), 'ranking': idx,'description':fname})
            datadict['搜尋詞'].append(kw)
            datadict['結果標題'].append(txt)
            datadict['結果網址'].append(href)
            datadict['結果名次'].append(str(idx))
            datadict['結果說明'].append(desc)

            # if len(txt) > 10:
            #     if domain is not None:
            #         for d in domain:
            #             if d in href:
            #                 print('found....')
            #                 print('clicked....')
            #                 print(href)
            #                 print(txt)
            #                 print("ranking", idx)
            #                 found = True
            #                 return
            #     else:
            #         ex = False
            #         for ee in exclude:
            #             if ee in href:
            #                 ex = True
            #         if not ex:
            #             test_lst.append(elmt)
            #             txt_dict[elmt] = txt

            idx += 1

        # if exclude is not None:
        #     print('exclude')
        #     elmt = random.choice(test_lst[5:])
        #     print(elmt)
        #     print(txt_dict[elmt])
        #
        #     webdriver.ActionChains(driver).move_to_element(elmt).perform()
        #     #            elmt.click()
        #     webdriver.ActionChains(driver).move_to_element(elmt).click().perform()
        #     time.sleep(5)
        #
        # if not found:
        #     table.insert(
        #         {'ranking': -1, 'kw': kw, 'results': numresults, 'url': '', 'title': '未收錄', 'descrption': desc})

        df['搜尋詞'] = datadict['搜尋詞']
        df['結果標題'] = datadict['結果標題']
        df['結果網址'] = datadict['結果網址']
        df['結果名次'] = datadict['結果名次']
        df['結果說明'] = datadict['結果說明']
        df.to_csv(r"C:\/Users\/s1301\/Downloads\/"+date+fname+".csv")




    except:
        print('exception')
        traceback.print_exc()
    #    time.sleep(9999)
    #    driver.save_screenshot('c:/tmp/seo/'+kw+".png")

    driver.quit()
    sys.exit()

d = {'ny':"https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWIJM6j_9LFNgPC_A_jo1ZldZjt50mA:1732591817687&ei=yUBFZ4vRKfChvr0Pv9CqkQQ&start=1&sa=N&sstk=ATObxK74c-Vmj60wMyZAkg6KYknAUt3xCeh6PWZn_0VWHs666zLLIqx32MESjFiY6pUy0nmojNGr9rdIBk7GqZETOYTma4FcqskAWg&ved=2ahUKEwjL2dW-h_mJAxXwkK8BHT-oKkIQ8tMDegQICRAE&biw=1274&bih=689&dpr=2.24",
     'wc':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICItV2FzaGluZ3RvbixEaXN0cmljdCBvZiBDb2x1bWJpYSxVbml0ZWQgU3RhdGVz&gws_rd=cr#gws_rd=cr&ip=1",
     'miami':"https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWIKPS2ivSJI7PP3_HRmy4H7dkLevCA:1732591895799&ei=F0FFZ7m_MKjZ1e8P9raGmQg&start=1&sa=N&sstk=ATObxK5Pv_BLMA5qsBfP9ByNdQYL8X07y597zLV907rx32LLXesARM2JfmOiSHWxmUJ07LOGiijKZVDzFt_Bdou4pzivtTd0I34uyA&ved=2ahUKEwj5pvXjh_mJAxWobPUHHXabIYMQ8tMDegQICxAE&biw=1274&bih=689&dpr=2.24",
     'chicago':"https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIeQ2hpY2FnbyxJbGxpbm9pcyxVbml0ZWQgU3RhdGVz&gws_rd=cr"}

d_3 = {
    'jp':'https://www.google.com/search?q=angelo+koo&hl=ja&gl=jp&num=100&uule=w+CAIQICIHTm9reW8sSmFwYW4',
        'ru':'https://www.google.com/search?q=angelo+koo&hl=ru&gl=ru&num=100&uule=w+CAIQICIGTW9zY293',
   'fr':'https://www.google.com/search?q=angelo+koo&hl=fr&gl=fr&num=100&uule=w+CAIQICIGUGFyaXM'}

location = 'fr'
run_once({'kw':'angelo koo','fname':location,'date':'0703','url':d_3[location]})

####手動截圖:須按右下角的設定選擇區域######
ny="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIWTmV3IFlvcmssVW5pdGVkIFN0YXRlcw&gws_rd=cr"
wc="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICItV2FzaGluZ3RvbixEaXN0cmljdCBvZiBDb2x1bWJpYSxVbml0ZWQgU3RhdGVz&gws_rd=cr#gws_rd=cr&ip=1"
miami="https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWIKPS2ivSJI7PP3_HRmy4H7dkLevCA:1732591895799&ei=F0FFZ7m_MKjZ1e8P9raGmQg&start=1&sa=N&sstk=ATObxK5Pv_BLMA5qsBfP9ByNdQYL8X07y597zLV907rx32LLXesARM2JfmOiSHWxmUJ07LOGiijKZVDzFt_Bdou4pzivtTd0I34uyA&ved=2ahUKEwj5pvXjh_mJAxWobPUHHXabIYMQ8tMDegQICxAE&biw=1274&bih=689&dpr=2.24"
chicago="https://www.google.com/search?q=angelo+koo&hl=en&gl=us&num=100&uule=w+CAIQICIeQ2hpY2FnbyxJbGxpbm9pcyxVbml0ZWQgU3RhdGVz&gws_rd=cr"

# d = {'ny':"https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWIJM6j_9LFNgPC_A_jo1ZldZjt50mA:1732591817687&ei=yUBFZ4vRKfChvr0Pv9CqkQQ&start=1&sa=N&sstk=ATObxK74c-Vmj60wMyZAkg6KYknAUt3xCeh6PWZn_0VWHs666zLLIqx32MESjFiY6pUy0nmojNGr9rdIBk7GqZETOYTma4FcqskAWg&ved=2ahUKEwjL2dW-h_mJAxXwkK8BHT-oKkIQ8tMDegQICRAE&biw=1274&bih=689&dpr=2.24",
#      'wc':"https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWILuIc3D0f7Biau98sqFUUZogBqCDQ:1732591880729&ei=CEFFZ4OWLLno1e8Pz8DawQE&start=1&sa=N&sstk=ATObxK7tiLCo3LSdvk03JNDl8jRF2TRafES_b3y_Zdu-tOEeei8BkqsWFkySS3UhrMToLj9XaJoMfO9cbVlcbpzBmHQBcmpw4NTxYw&ved=2ahUKEwiDut3ch_mJAxU5dPUHHU-gNhgQ8tMDegQIDRAE&biw=1274&bih=689&dpr=2.24",
#      'miami':"https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWIKPS2ivSJI7PP3_HRmy4H7dkLevCA:1732591895799&ei=F0FFZ7m_MKjZ1e8P9raGmQg&start=1&sa=N&sstk=ATObxK5Pv_BLMA5qsBfP9ByNdQYL8X07y597zLV907rx32LLXesARM2JfmOiSHWxmUJ07LOGiijKZVDzFt_Bdou4pzivtTd0I34uyA&ved=2ahUKEwj5pvXjh_mJAxWobPUHHXabIYMQ8tMDegQICxAE&biw=1274&bih=689&dpr=2.24",
#      'chicago':"https://www.google.com/search?q=angelo+koo&num=100&sca_esv=a67a64356a29d00d&hl=en&gl=us&sxsrf=ADLYWILJICQpK9Ihu-LtB3GE2lKjcKpsjQ:1732591304207&ei=yD5FZ5iyDPbq1e8PuM3msAU&start=1&sa=N&sstk=ATObxK7FwISMQrob8SCRdfsBYNwoBYcZNm8OzwvSVkL1SgQphssf9htXgWrFb0IS39QEWx_4yuAOLwF4FszOrem0NnpL_n-dJrgjH5OB9rtGSRm7ykBoy-ju3sLbo_wUL_Kd&ved=2ahUKEwiYtunJhfmJAxV2dfUHHbimGVY4yAEQ8tMDegQIDBAJ&biw=1275&bih=689&dpr=2.24"}