2021-09-07_Hover,ScrollAndClick_Selenium.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cfd80f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "import chromedriver_binary\n",
    "from time import sleep\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "import re\n",
    "\n",
    "url = \"https://spi21.mapyourshow.com/8_0/floorplan/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c7c8e53e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "472\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "import chromedriver_binary\n",
    "from time import sleep\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "import re\n",
    "import time\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.common.exceptions import StaleElementReferenceException\n",
    "\n",
    "url = 'https://spi21.mapyourshow.com/8_0/explore/exhibitor-alphalist.cfm#/'\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('headless')  \n",
    "driver = webdriver.Chrome()\n",
    "driver.get(url)\n",
    "\n",
    "time.sleep(3)\n",
    "\n",
    "#need to scroll down\n",
    "driver.execute_script(\"window.scrollTo(20, 40);\")\n",
    "\n",
    "time.sleep(5)\n",
    "\n",
    "#click see all results\n",
    "idName = \"exhibitor-results\"\n",
    "# xPath = \"//section[@id = '%s']/div/span[1]\"%(idName)\n",
    "xPath = \"//*[@id = '%s']/div[1]/span[1]/a/span\"%(idName)\n",
    "#wait until there is any element to hover\n",
    "wait = WebDriverWait(driver, 20)\n",
    "auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, xPath)))\n",
    "element = driver.find_element_by_xpath(xPath)\n",
    "element.click()\n",
    "\n",
    "time.sleep(5)\n",
    "\n",
    "xPath = '//a'\n",
    "elements = driver.find_elements_by_xpath(xPath)\n",
    "#why page content is not the same after selenium?\n",
    "hrefList = [i.get_attribute(\"href\") for i in elements]\n",
    "\n",
    "nameList = [i.get_attribute(\"innerText\") for i in elements]\n",
    "\n",
    "\n",
    "#result\n",
    "resultList = []\n",
    "for no, href in enumerate(hrefList):\n",
    "    if re.search(\"exhid=\\d+$\", str(href)):\n",
    "        resultList.append([nameList[no], href])\n",
    "print(len(resultList))\n",
    "\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9d72d411",
   "metadata": {},
   "outputs": [],
   "source": [
    "finalHrefList = set([i[1] for i in resultList])\n",
    "\n",
    "filename = 'C:/Users/JiunShyanGoh/Documents/Upwork/DataScraping/Data/HrefList.csv'\n",
    "\n",
    "with open(filename, 'w') as filehandle:\n",
    "    for href in finalHrefList:\n",
    "        filehandle.write('%s\\n' %href)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5f41e617",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'C:/Users/JiunShyanGoh/Documents/Upwork/DataScraping/Data/HrefList.csv'\n",
    "\n",
    "a = []\n",
    "with open(filename, 'r') as filehandle:\n",
    "    for line in filehandle:\n",
    "        a.append(re.sub('\\n','',line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bb89efc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://spi21.mapyourshow.com/8_0/exhibitor/exhibitor-details.cfm?exhid=8799734'\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('headless')  \n",
    "driver = webdriver.Chrome(options = options)\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fc7331b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "xPath = \"//div[@class = 'wrapper-centered']\"\n",
    "elements = driver.find_elements_by_xpath(xPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0087d30f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Contact Us\\nCONTACT US\\nAddress: 610 A East Battlefield Road\\nAddress 2: PMB 247\\nCity: Springfield\\nState: MO\\nCountry: United States of America\\nZip: 65807\\nPhone: 5415161100\\nWebsite: www.cascadeecominerals.com',\n",
       " 'Booths\\nFirst Floor Exhibit Hall — 2171',\n",
       " 'About Cascade Eco Minerals\\nCascade Eco Minerals utilizes state-of-the-art technology, innovative processes, and proprietary offerings to provide a comprehensive domestic recycling solution for the Solar Industry. With over 80 years of experience in glass processing and a mission to grow the circular economy, you’ll find no greater partner in solar glass recycling, than CEM.',\n",
       " 'Notes\\nCreate or Log in to My Show Planner to add notes.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[i.text for i in elements]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "32faf9fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Manufacturer - PV Cell', 'Manufacturer - PV Module']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://spi21.mapyourshow.com/8_0/exhibitor/exhibitor-details.cfm?exhid=7088'\n",
    "# url = 'https://spi21.mapyourshow.com/8_0/exhibitor/exhibitor-details.cfm?exhid=8799734'\n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('headless')  \n",
    "driver = webdriver.Chrome(options = options)\n",
    "driver.get(url)\n",
    "\n",
    "#need to scroll down\n",
    "driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "[i.text for i in driver.find_elements_by_xpath(\"//h2/a\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5113ab7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join([i.text for i in driver.find_elements_by_xpath(\"//h2/a\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e4fbbca",
   "metadata": {},
   "outputs": [],
   "source": [
    "#number of seconds to sleep\n",
    "timeSleep = 3\n",
    "\n",
    "#get usernamelist and name list \n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('headless')  \n",
    "driver = webdriver.Chrome()\n",
    "driver.get(url)\n",
    "\n",
    "#wait for canvas of floor to show up\n",
    "canvasClassName = \"ol-unselectable\"\n",
    "xPath = \"//canvas[@class = '%s']\"%canvasClassName\n",
    "#wait until there is any element to hover\n",
    "wait = WebDriverWait(driver, 20)\n",
    "auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, xPath)))\n",
    "\n",
    "# clickableElement\n",
    "#with position (not working yet)\n",
    "leftPosition = \"139px\"\n",
    "topPosition = \"179px\"\n",
    "styleValue = \"position: absolute; left: %s; top: %s\"%(leftPosition, topPosition)\n",
    "className = \"ol-overlay-container ol-selectable\"\n",
    "xPath = \"//div[@class = '%s']\"%className +  \"[@style = '%s\"%styleValue + \\\n",
    "\"; display: none;']\"\n",
    "\n",
    "#this work\n",
    "className = 'mys-floorplan-canvas'\n",
    "xPath = \"//div[@class = '%s']\"%className\n",
    "clickableElement = driver.find_element_by_xpath(xPath)\n",
    "action = ActionChains(driver)\n",
    "action.move_to_element(clickableElement).click().perform()\n",
    "\n",
    "\n",
    "#company name\n",
    "resultDict = {}\n",
    "\n",
    "className = 'flex  items-center  mb3  mb0-m  flex-wrap'\n",
    "xPath = \"//div[@class = '%s']\"%className\n",
    "auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, xPath)))\n",
    "element = driver.find_element_by_xpath(xPath)\n",
    "#alternative: companyName = element.text\n",
    "companyName = element.get_attribute(\"innerText\")\n",
    "resultDict['Name'] = companyName\n",
    "\n",
    "#address, city, state, country, zip, phone, website\n",
    "elements = driver.find_elements_by_xpath(\"//div[@class = 'wrapper-centered']\")\n",
    "textList = [element.text for element in elements]\n",
    "#fields in the first element of textList\n",
    "fieldList = ['Address', 'City', 'State', 'Country', 'Zip', 'Phone', 'Website']\n",
    "for field in fieldList:\n",
    "    resultDict[field] = re.search(\"%s:(.*)\\n*\"%field, textList[0]).group(1).strip()\n",
    "        \n",
    "#booth\n",
    "resultDict['Booth'] = textList[1].split('\\n')[1]\n",
    "\n",
    "#about\n",
    "resultDict['About'] = textList[2].split('\\n')[1]\n",
    "\n",
    "#product categories\n",
    "resultDict['Product Categories'] = textList[3].split('\\n')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "018697fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "companyName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f69d56b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#number of seconds to sleep\n",
    "timeSleep = 3\n",
    "\n",
    "#get usernamelist and name list \n",
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('headless')  \n",
    "driver = webdriver.Chrome()\n",
    "driver.get(url)\n",
    "\n",
    "#wait for canvas of floor to show up\n",
    "canvasClassName = \"ol-overlaycontainer-stopevent\"\n",
    "xPath = \"//div[@class = '%s']\"%canvasClassName\n",
    "#wait until there is any element to hover\n",
    "wait = WebDriverWait(driver, 20)\n",
    "auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, xPath)))\n",
    "\n",
    "leftPosition = \"547px\"\n",
    "topPosition = \"377px\"\n",
    "styleValue = \"position: absolute; left: %s; top: %s;\"%(leftPosition, topPosition)\n",
    "className = \"ol-overlay-container ol-selectable\"\n",
    "xPath = \"//div[@class = '%s']\"%className +  \"[@style = '%s']\"%styleValue \n",
    "auto_element = wait.until(EC.visibility_of_element_located((By.XPATH, xPath)))\n",
    "clickableElement = driver.find_element_by_xpath(xPath)\n",
    "action = ActionChains(driver)\n",
    "action.move_to_element(clickableElement).click().perform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52e16a51",
   "metadata": {},
   "outputs": [],
   "source": [
    "xPath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "191dc97b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}