2021-09-01_LinkedInScrapingFinal.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f3b9aaa2",
   "metadata": {},
   "source": [
    "### Use package = scrape_linkedin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bfeddfcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from time import sleep\n",
    "from selenium import webdriver\n",
    "import chromedriver_binary\n",
    "from scrape_linkedin import ProfileScraper\n",
    "import re\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d82951d",
   "metadata": {},
   "source": [
    "### Pre define parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "204b3e50",
   "metadata": {},
   "outputs": [],
   "source": [
    "#created a linkedin profile: yeomangoh@yahoo.com, password:08075323\n",
    "#get li_at cookie (need login): https://www.smartwriter.ai/university/how-to-access-your-linkedin-cookie-and-what-exactly-does-a-cookie-do\n",
    "li_at_value = 'AQEDATd4RK4C9QUEAAABe5xdggMAAAF7wGoGA1YAP8VFRmt1hVKulFeHCP-2ztYgJis24Yuq_Bxd7-tMlg8g3NG06cHybRecFh3tJjtGKT2l8s3fWVbctPC2z22jnR0oUFjhuL0-c0fF8FPzku2jlYTx'\n",
    "\n",
    "#get names from organization page\n",
    "companyUrl = \"https://healthaxis.com/leadership-team\"\n",
    "\n",
    "companyName = \"HealthAxis Group\"\n",
    "\n",
    "#number of col for final table\n",
    "colLen = 8\n",
    "\n",
    "timeSleep = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15d703ba",
   "metadata": {},
   "source": [
    "### UDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "48625e32",
   "metadata": {},
   "outputs": [],
   "source": [
    "def useProfileScraper(userName):\n",
    "    try:\n",
    "        with ProfileScraper(cookie = li_at_value) as scraper:\n",
    "            profile = scraper.scrape(user ='%s'%userName)\n",
    "\n",
    "        profileDict = profile.to_dict()\n",
    "\n",
    "        #personal info\n",
    "        colList = ['phone', 'email', 'company', 'school']\n",
    "        resultDict = {}\n",
    "        for col in colList:\n",
    "            if col in profileDict['personal_info'].keys():\n",
    "                resultDict[col] = profileDict['personal_info'][col]\n",
    "\n",
    "        #working exp\n",
    "        #skill\n",
    "        resultDict['skills'] = ','.join([i['name'] for i in profileDict['skills']])\n",
    "\n",
    "        #username\n",
    "        resultDict['username'] = username\n",
    "        \n",
    "        #store result\n",
    "        return resultDict\n",
    "    \n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36da41d4",
   "metadata": {},
   "source": [
    "### Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "c2d86f10",
   "metadata": {},
   "outputs": [],
   "source": [
    "#get usernamelist and name list \n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('headless')  \n",
    "driver = webdriver.Chrome(options = options)\n",
    "driver.get(companyUrl)\n",
    "\n",
    "className = \"wpb_text_column wpb_content_element \"\n",
    "a_Tag = driver.find_elements_by_xpath(\"//div[@class = '%s']/div/p/span/a\"%className)\n",
    "FirstSpan_Tag = driver.find_elements_by_xpath(\"//div[@class = '%s']/div/p/span[1]\"%className)\n",
    "# SecondSpan_Tag = driver.find_elements_by_xpath(\"//div[@class = '%s']/div/p/span[@style = 'font-size: 18px;']\"%className)\n",
    "SecondSpan_Tag = driver.find_elements_by_xpath(\"//div[@class = '%s']/div/p/span[2]\"%className)\n",
    "hrefList = [i.get_attribute(\"href\") for i in a_Tag]\n",
    "\n",
    "#get nameList and jobList\n",
    "nameList = [i.get_attribute(\"innerText\") for i in a_Tag]\n",
    "jobTitleList = [i.get_attribute(\"innerText\") for i in SecondSpan_Tag]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "cbc58b5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['name', 'phone', 'email', 'company', 'school', 'current title',\n",
       "       'skills', 'username'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfResult.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "df5afaa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "resultList = []\n",
    "\n",
    "for no, name in enumerate(nameList):\n",
    "    #search and click\n",
    "    url = 'https://www.linkedin.com'\n",
    "    options = webdriver.ChromeOptions()\n",
    "    options.add_argument('headless')  \n",
    "    driver = webdriver.Chrome()\n",
    "    driver.get(url)\n",
    "    # driver.add_cookie({\"name\":\"li_at\", \"value\":li_at_value})\n",
    "\n",
    "    email = \"yeomangoh@yahoo.com\"\n",
    "    password = \"08075323\"\n",
    "    #key in email and password\n",
    "    emailBox = driver.find_element_by_xpath(\"//*[@id = 'session_key']\")\n",
    "    emailBox.send_keys(email)\n",
    "    sleep(timeSleep)\n",
    "    passwordBox = driver.find_element_by_xpath(\"//*[@id = 'session_password']\")\n",
    "    passwordBox.send_keys(password)\n",
    "\n",
    "    #click sign in button\n",
    "    sleep(timeSleep)\n",
    "    signInButton = driver.find_element_by_xpath(\"//button[@type = 'submit']\")\n",
    "    signInButton.click()\n",
    "\n",
    "    #key in text\n",
    "    sleep(timeSleep)\n",
    "    searchText = name + ' ' + companyName\n",
    "    inputBox_xPath = \"//input[@placeholder = 'Search']\"\n",
    "    wait = WebDriverWait(driver, 10)\n",
    "    auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, inputBox_xPath)))\n",
    "    inputBox = driver.find_element_by_xpath(inputBox_xPath)\n",
    "    inputBox.send_keys(\"%s\"%searchText)\n",
    "    inputBox.send_keys(Keys.RETURN)\n",
    "\n",
    "    #click result link\n",
    "    sleep(timeSleep)\n",
    "    resultLink_xPath = \"//a[@class = 'app-aware-link']\"\n",
    "    try:\n",
    "        #here may fail if no results even fuzzy\n",
    "        wait = WebDriverWait(driver, 10)\n",
    "        auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, resultLink_xPath)))\n",
    "        resultLink = driver.find_element_by_xpath(resultLink_xPath)\n",
    "        # resultLink.click()\n",
    "\n",
    "        #get url from href to pass to scrape_linkedin func:\n",
    "        currentUrl = resultLink.get_attribute(\"href\")\n",
    "        username = re.search(\".*/(.*)\\?\", currentUrl).group(1)\n",
    "\n",
    "    except:\n",
    "        #if not even fuzzy match\n",
    "        currentUrl = ''\n",
    "        username = ''\n",
    "\n",
    "    #if no exact match (return results/all) or not even fuzzy match (username = '')\n",
    "    if re.search(\"results\\/all\", currentUrl) or username == '':\n",
    "        tempDict = {}\n",
    "        tempDict['username'] = username\n",
    "        tempDict['name'] = name\n",
    "        tempDict['jobTitle'] = jobTitleList[no]\n",
    "        for col in ['phone', 'email', 'company', 'school', 'skills']:\n",
    "            tempDict[col] = ''\n",
    "        resultList.append(tempDict)\n",
    "    else:\n",
    "        tempDict = useProfileScraper(username)\n",
    "        tempDict['name'] = name\n",
    "        tempDict['jobTitle'] = jobTitleList[no]\n",
    "        resultList.append(tempDict)\n",
    "\n",
    "    driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "0d41216b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>phone</th>\n",
       "      <th>email</th>\n",
       "      <th>company</th>\n",
       "      <th>school</th>\n",
       "      <th>skills</th>\n",
       "      <th>username</th>\n",
       "      <th>name</th>\n",
       "      <th>jobTitle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>The HealthAxis Group</td>\n",
       "      <td>Columbia University - Columbia Business School</td>\n",
       "      <td>Start-ups,Business Development,Management,Entr...</td>\n",
       "      <td>shilenpatel</td>\n",
       "      <td>Shilen Patel</td>\n",
       "      <td>Chairman &amp; CEO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>HealthAxis Group</td>\n",
       "      <td>State University of New York College at Potsdam</td>\n",
       "      <td></td>\n",
       "      <td>jason-patchen-084062145</td>\n",
       "      <td>Jason Patchen</td>\n",
       "      <td>President</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>HealthAxis Group</td>\n",
       "      <td>Florida State University - College of Business</td>\n",
       "      <td>Start-ups,Management,Business Development,New ...</td>\n",
       "      <td>jimclark01</td>\n",
       "      <td>Jim Clark</td>\n",
       "      <td>Chief Growth Officer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>HealthAxis Group</td>\n",
       "      <td>West Virginia Wesleyan College</td>\n",
       "      <td>Process Improvement,Business Analysis,Software...</td>\n",
       "      <td>zacharykilburn</td>\n",
       "      <td>Zachary Kilburn</td>\n",
       "      <td>Chief Technology Officer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Michele Mahoney</td>\n",
       "      <td>Chief Healthcare Operations Officer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Eric Strikowski</td>\n",
       "      <td>Chief Innovation Officer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Lisa Davis</td>\n",
       "      <td>Chief Data Officer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>HealthAxis Group</td>\n",
       "      <td>Western New England College School of Law</td>\n",
       "      <td>Medicare,Medicaid,Policy,Insurance,Health Insu...</td>\n",
       "      <td>michael-friel-268b6757</td>\n",
       "      <td>Michael Friel</td>\n",
       "      <td>Compliance Officer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  phone email               company  \\\n",
       "0  None  None  The HealthAxis Group   \n",
       "1  None  None      HealthAxis Group   \n",
       "2  None  None      HealthAxis Group   \n",
       "3  None  None      HealthAxis Group   \n",
       "4                                     \n",
       "5                                     \n",
       "6                                     \n",
       "7  None  None      HealthAxis Group   \n",
       "\n",
       "                                            school  \\\n",
       "0   Columbia University - Columbia Business School   \n",
       "1  State University of New York College at Potsdam   \n",
       "2   Florida State University - College of Business   \n",
       "3                   West Virginia Wesleyan College   \n",
       "4                                                    \n",
       "5                                                    \n",
       "6                                                    \n",
       "7        Western New England College School of Law   \n",
       "\n",
       "                                              skills                 username  \\\n",
       "0  Start-ups,Business Development,Management,Entr...              shilenpatel   \n",
       "1                                                     jason-patchen-084062145   \n",
       "2  Start-ups,Management,Business Development,New ...               jimclark01   \n",
       "3  Process Improvement,Business Analysis,Software...           zacharykilburn   \n",
       "4                                                                               \n",
       "5                                                                               \n",
       "6                                                                               \n",
       "7  Medicare,Medicaid,Policy,Insurance,Health Insu...   michael-friel-268b6757   \n",
       "\n",
       "              name                             jobTitle  \n",
       "0     Shilen Patel                       Chairman & CEO  \n",
       "1    Jason Patchen                            President  \n",
       "2        Jim Clark                 Chief Growth Officer  \n",
       "3  Zachary Kilburn             Chief Technology Officer  \n",
       "4  Michele Mahoney  Chief Healthcare Operations Officer  \n",
       "5  Eric Strikowski             Chief Innovation Officer  \n",
       "6       Lisa Davis                   Chief Data Officer  \n",
       "7    Michael Friel                   Compliance Officer  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfResult = pd.DataFrame(resultList)\n",
    "dfResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d6c9a060",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"Data/2021-09-01_LinkedInExample.csv\"\n",
    "dfResult.to_csv(filename, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6e9417e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#possible side to find healthcare companies in ohio\n",
    "#https://insurancecareers.org/ohio-companies"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ee0d274",
   "metadata": {},
   "source": [
    "### Past Codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d4077b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# resultList = []\n",
    "# failedList = []\n",
    "# for no, username in enumerate(userNameList):\n",
    "#     result = useProfileScraper(username)\n",
    "#     if isinstance(result, dict):\n",
    "#         resultList.append(result)\n",
    "#     else:\n",
    "#         failedList.append(nameList[no])\n",
    "        \n",
    "# #form df\n",
    "# dfResult = pd.DataFrame(resultList)\n",
    "\n",
    "# #get result where not match company - not the person\n",
    "# notMatchList = list(dfResult[dfResult['company'].map(lambda x:re.search(companyName, x) == None)]\n",
    "#                     ['name'].values)\n",
    "\n",
    "# failedList2 = failedList + notMatchList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "33887115",
   "metadata": {},
   "outputs": [
    {
     "ename": "TimeoutException",
     "evalue": "Message: \n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTimeoutException\u001b[0m                          Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-30-45ad3f444203>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     29\u001b[0m     \u001b[0minputBox_xPath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"//input[@placeholder = 'Search']\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m     \u001b[0mwait\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mWebDriverWait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m     \u001b[0mauto_element\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwait\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muntil\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mEC\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvisibility_of_element_located\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXPATH\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputBox_xPath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     32\u001b[0m     \u001b[0minputBox\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputBox_xPath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m     \u001b[0minputBox\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_keys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"%s\"\u001b[0m\u001b[1;33m%\u001b[0m\u001b[0msearchText\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\support\\wait.py\u001b[0m in \u001b[0;36muntil\u001b[1;34m(self, method, message)\u001b[0m\n\u001b[0;32m     78\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mend_time\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 80\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0muntil_not\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTimeoutException\u001b[0m: Message: \n"
     ]
    }
   ],
   "source": [
    "resultList2 = []\n",
    "\n",
    "for name in failedList2:\n",
    "    #search and click\n",
    "    url = 'https://www.linkedin.com'\n",
    "    options = webdriver.ChromeOptions()\n",
    "    options.add_argument('headless')  \n",
    "    driver = webdriver.Chrome()\n",
    "    driver.get(url)\n",
    "    # driver.add_cookie({\"name\":\"li_at\", \"value\":li_at_value})\n",
    "\n",
    "    email = \"yeomangoh@yahoo.com\"\n",
    "    password = \"08075323\"\n",
    "    #key in email and password\n",
    "    emailBox = driver.find_element_by_xpath(\"//*[@id = 'session_key']\")\n",
    "    emailBox.send_keys(email)\n",
    "    sleep(timeSleep)\n",
    "    passwordBox = driver.find_element_by_xpath(\"//*[@id = 'session_password']\")\n",
    "    passwordBox.send_keys(password)\n",
    "\n",
    "    #click sign in button\n",
    "    sleep(timeSleep)\n",
    "    signInButton = driver.find_element_by_xpath(\"//button[@type = 'submit']\")\n",
    "    signInButton.click()\n",
    "\n",
    "    #key in text\n",
    "    sleep(timeSleep)\n",
    "    searchText = name + ' ' + companyName\n",
    "    inputBox_xPath = \"//input[@placeholder = 'Search']\"\n",
    "    wait = WebDriverWait(driver, 10)\n",
    "    auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, inputBox_xPath)))\n",
    "    inputBox = driver.find_element_by_xpath(inputBox_xPath)\n",
    "    inputBox.send_keys(\"%s\"%searchText)\n",
    "    inputBox.send_keys(Keys.RETURN)\n",
    "\n",
    "    #click result link\n",
    "    sleep(timeSleep)\n",
    "    resultLink_xPath = \"//a[@class = 'app-aware-link']\"\n",
    "    try:\n",
    "        #here may fail if no results even fuzzy\n",
    "        wait = WebDriverWait(driver, 10)\n",
    "        auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, resultLink_xPath)))\n",
    "        resultLink = driver.find_element_by_xpath(resultLink_xPath)\n",
    "        # resultLink.click()\n",
    "\n",
    "        #get url from href to pass to scrape_linkedin func:\n",
    "        currentUrl = resultLink.get_attribute(\"href\")\n",
    "        username = re.search(\".*/(.*)\\?\", currentUrl).group(1)\n",
    "\n",
    "    except:\n",
    "        #if not even fuzzy match\n",
    "        currentUrl = ''\n",
    "        username = ''\n",
    "\n",
    "    #if no exact match (return results/all) or not even fuzzy match (username = '')\n",
    "    if re.search(\"results\\/all\", currentUrl) or username == '':\n",
    "        tempDict = {}\n",
    "        tempDict['username'] = username\n",
    "        tempDict['name'] = name\n",
    "        for col in [i for i in dfResult.columns if i not in ['name', 'username']]:\n",
    "            tempDict[col] = ''\n",
    "        resultList2.append(tempDict)\n",
    "    else:\n",
    "        resultList2.append(useProfileScraper(username))\n",
    "\n",
    "    driver.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}