How To Retrieve Google Search Results Into A DataFrame For Analysing With Data Science Skills Using Python

import advertools as adv
cs_id = "a5f90a3b5a88e0d0d"
cs_key = "AIzaSyCQhrSpIr8LFRPL6LhFfL9K59Gqr0dhK5c"
adv.serp_goog(q='Holistic SEO', gl=['tr', 'fr'],cr=["countryTR"], cx=cs_id, key=cs_key)

df  = adv.serp_goog(q='Holistic SEO', gl=['tr', 'fr'],cr=["countryTR"], cx=cs_id, key=cs_key)
df.shape

from termcolor import colored
for column20, column40, column60, column90  in zip(df.columns[:20], df.columns[20:40],df.columns[40:60],df.columns[60:90]):
    print(colored(f'{column:<12}', "green"), colored(f'{column40:<15}', "yellow"), colored(f'{column60:<32}', "white"),colored(f'{column90:<12}', "red"))

import pandas as pd
pd.set_option("display.max_colwidth",None)
df[['gl', 'rank', 'title', 'link']]

df_tr = df[df['gl'] == "tr"]
df_fr = df[df['gl'] == 'fr']

df = adv.serp_goog(q='Who is Abraham Lincoln?', gl=['us'], cr=["countryUS"], cx=cs_id, key=cs_key, start=[1,11,21,31,41,51,61,71,81,91])

pd.set_option("display.max_rows",None)
df.head(100)

df = adv.serp_goog(q=["Who is Abraham Lincoln?", "Who are the rivals of Abraham Lincoln?", "What did Abraham Lincoln do?", "Was Abraham Lincoln a Freemason"], gl=['us'], cr=["countryUS"], cx=cs_id, key=cs_key, start=[1,11,21,31,41,51,61,71,81,91])

df[df.docauthor.notna() == True][["docauthor", "htmlTitle", "snippet"]]

df[df['books:author'].isna() == False][['books:author', "htmlTitle", "snippet"]]

key = "AIzaSyAPQD4WDYAIkRlPYAdFml3jtUaICW6P9ZE"
adv.knowledge_graph(key=key, query=['Doris Kearns Goodwin', "Louis Dale Carman"])[['result.name', 'result.description', 'resultScore']].style.background_gradient(cmap='Blues')

(df.pivot_table('rank', 'displayLink', 
aggfunc=['count', 'mean']).sort_values([('count', 'rank'), ('mean', 'rank')], 
ascending=[False, True]).assign(coverage=lambda df: df[('count', 'rank')] / len(df)*10).head(10).style.format({("coverage", ''): "{:.1%}", ('mean', 'rank'): '{:.2f}'})).background_gradient(cmap="viridis")

df[df['displayLink'] == "en.wikipedia.org"]

df['displayLink'] = df['displayLink'].str.replace("www.", "")
df_average = df.pivot_table("rank", "displayLink", aggfunc=["count", "mean"]).sort_values([('count', 'rank'), ('mean', 'rank')], 
ascending=[False, True])
df_average.sort_values(("count", "rank"), ascending=False)[:10].plot(kind="bar", rot=0, figsize=(25,10), fontsize=10, grid=True, xlabel="Domain Names", title="Average Rank and Rank Amount", position=0.5)

import matplotlib.pyplot as plt 
from matplotlib.cm import tab10
from matplotlib.ticker import EngFormatter
queries = ["Who is Abraham Lincoln?", "Who are the rivals of Abraham Lincoln?", "What did Abraham Lincoln do?", "Was Abraham Lincoln a Freemason"]
fig, ax = plt.subplots(4,1, facecolor='#eeeeee')
fig.set_size_inches(10, 10)
for i in range(4):
    ax[i].set_frame_on(False)
    ax[i].barh((df[df['searchTerms']==queries[i]])['displayLink'][:5], df[df['searchTerms']== queries[i]]['rank'][:5],color=tab10.colors[i+5])
    ax[i].set_title('Ranking Results for Top 5 Domain for ' + "'" + queries[i] + "'", fontsize=15)
    ax[i].tick_params(labelsize=12)
    ax[i].xaxis.set_major_formatter(EngFormatter())
 
plt.tight_layout()
fig.savefig(fname=”serp.png”, dpi=300, quality=100)
plt.show()

fig.data = []
fig = go.Figure()
for i, domain in enumerate(df_graph_first_10["Domain"]):
    filtered_graph = df_graph_first_10[df_graph_first_10["Domain"]==domain]
    fig.add_scatter(x=filtered_graph["Query"], y=filtered_graph["Rank"], name=domain, text=[filtered_graph["Domain"]])
 
fig.update_layout(width=1400, height=800, legend=dict(
    yanchor="top",
    y=1.5,
    xanchor="right",
    x=1.3),
    hovermode="x unified", title="SERP Changes According to Queries per Domain", title_font_family="Open Sans", title_font_size=25)
fig.update_xaxes(title="Queries", title_font={"size":20, "family":"Open Sans"})
fig.update_yaxes(title="Rankings", title_font={"size":20, "family":"Open Sans"})
fig.write_html("serp-check.html")
fig.show()

(df.to_csv("df_after_core_algorithm_update.csv")

df = adv.serp_goog(q=["Who is Abraham Lincoln?", "Who are the rivals of Abraham Lincoln?", "What did Abraham Lincoln do?", "Was Abraham Lincoln a Freemason"], gl=['us'], cr=["countryUS"], cx=cs_id, key=cs_key, start=[1,11,21,31,41,51,61,71,81,91], dateRestrict='w5', exactTerms="Mary Todd Lincoln", excludeTerms="kill", hq=["the president","childhood"])

df_graph_10 = df_graph[df_graph["Rank"]<=10]

queries = ["Who is Abraham Lincoln? site:wikipedia.org", "Who are the rivals of Abraham Lincoln? -wikipedia.org", "What did Abraham Lincoln do? intitle:President", "Was Abraham Lincoln a Freemason inurl:kill"]
df = adv.serp_goog(q="Who killed Abraham Lincoln site:wikipedia.org", cx=cs_id, key=cs_key)

df[df["searchTerms"]==queries[0]]["displayLink"].str.contains("wikipedia")

df[df["searchTerms"]==queries[0]][["displayLink","searchTerms", "title", "link"]]

df[df["searchTerms"]==queries[1]]["displayLink"].str.contains("wikipedia")

df[df["searchTerms"]==queries[1]][["displayLink","searchTerms", "title", "link"]]

df[df["searchTerms"]==queries[2]]["title"].str.contains("President", case=False)

df[df["searchTerms"]==queries[3]]["link"].str.contains("kill", case=False)