<filename>targets.py
baseurl = "https://www.boots.com/"
page_string = "?pageNo="

#~ boots categories
categories = ["health-pharmacy/medicines-treatments/painrelief/period-pain-tablets",
              "health-pharmacy/medicines-treatments/painrelief/all-painrelief",
              "health-pharmacy/medicines-treatments/eye-care",
              "health-pharmacy/medicines-treatments/digestion",
              "health-pharmacy/medicines-treatments/heartburn-indigestion-",
              "health-pharmacy/medicines-treatments/footcare",
              "health-pharmacy/medicines-treatments/allergy-hayfever",
              "health-pharmacy/medicines-treatments/skin-problems",
              "health-pharmacy/medicines-treatments/first-aid",
              "health-pharmacy/medicines-treatments/cold-flu-medication",
              "health-pharmacy/medicines-treatments/mouth-oral-care",
              "health-pharmacy/medicines-treatments/sleep",
              "health-pharmacy/medicines-treatments/ear-care",
              "health-pharmacy/medicines-treatments/mens-hair-loss",
              "health-pharmacy/medicines-treatments/diabetes",
              "health-pharmacy/medicines-treatments/healthyheart",
              "health-pharmacy/vitaminsandsupplements/vitamins-supplements-shop-by-ingredient",
              "health-pharmacy/lifestyle-wellbeing/familyplanning/pregnancy-tests",
              "health-pharmacy/lifestyle-wellbeing/familyplanning/maternity-tens-machine",
              "health-pharmacy/lifestyle-wellbeing/familyplanning/ovulation-fertility-tests",
              "health-pharmacy/lifestyle-wellbeing/familyplanning/pre-conception-lubricants",
              "health-pharmacy/lifestyle-wellbeing/familyplanning/pregnancysupplements",
              "health-pharmacy/health-value-packs-and-bundles",
              "health-pharmacy/covid-19-information-products-and-testing/thermometers",
              "health-pharmacy/covid-19-information-products-and-testing/immunehealth",
              "health-pharmacy/covid-19-information-products-and-testing/cold-flu-medication",
              "health-pharmacy/covid-19-information-products-and-testing/hand-sanitiser-antibacterial-cleaners-disinfectants",
              "health-pharmacy/covid-19-information-products-and-testing/skin-problems",
              "health-pharmacy/covid-19-information-products-and-testing/multivitamins",
              "health-pharmacy/covid-19-information-products-and-testing/baby-child-vitamins",
              "health-pharmacy/covid-19-information-products-and-testing/sleep",
              "health-pharmacy/covid-19-information-products-and-testing/first-aid",
              "health-pharmacy/reusable-and-disposable-face-masks",
              "health-pharmacy/hand-sanitiser-antibacterial-cleaners-disinfectants",
              "health-pharmacy/vitaminsandsupplements/womenshealth-supplements",
              "health-pharmacy/vitaminsandsupplements/multivitamins",
              "health-pharmacy/vitaminsandsupplements/immunehealth",
              "health-pharmacy/vitaminsandsupplements/pregnancysupplements",
              "health-pharmacy/vitaminsandsupplements/cannabidiol-cbd-oil",
              "health-pharmacy/vitaminsandsupplements/beautysupplements",
              "health-pharmacy/vitaminsandsupplements/jointhealth",
              "health-pharmacy/vitaminsandsupplements/hair-health-vitamins",
              "health-pharmacy/vitaminsandsupplements/digestive-health",
              "health-pharmacy/vitaminsandsupplements/menshealth-supplements",
              "health-pharmacy/vitaminsandsupplements/50plus-multivitamins",
              "health-pharmacy/vitaminsandsupplements/supplements-for-energy",
              "health-pharmacy/vitaminsandsupplements/eyehealth",
              "health-pharmacy/vitaminsandsupplements/bonehealth",
              "health-pharmacy/vitaminsandsupplements/brainhealth",
              "health-pharmacy/vitaminsandsupplements/vegan-vitamins",
              "health-pharmacy/baby-child-health/baby-child-vitamins",
              "health-pharmacy/baby-child-health/fever-pain-relief",
              "health-pharmacy/baby-child-health/child-cough-cold-flu",
              "health-pharmacy/baby-child-health/skincare-conditions",
              "health-pharmacy/baby-child-health/teething",
              "health-pharmacy/baby-child-health/allergy-hayfever-children",
              "health-pharmacy/baby-child-health/child-first-aid",
              "health-pharmacy/baby-child-health/nits-lice-worms",
              "health-pharmacy/baby-child-health/colic",
              "health-pharmacy/womenshealth/intimate-dryness--1",
              "health-pharmacy/womenshealth/vaginitis",
              "health-pharmacy/womenshealth/female-incontinence",
              "health-pharmacy/womenshealth/feminine-wash-wipes",
              "health-pharmacy/womenshealth/thrush",
              "health-pharmacy/womenshealth/menopause",
              "health-pharmacy/womenshealth/womenshealth-supplements",
              "health-pharmacy/womenshealth/cystitis-urinary-tract-infections",
              "health-pharmacy/womenshealth/womens-hair-loss",
              "health-pharmacy/womenshealth/period-pain",
              "health-pharmacy/womenshealth/womens-alternative-therapy",
              "health-pharmacy/menshealth/mens-sexual-health",
              "health-pharmacy/menshealth/male-incontinence",
              "health-pharmacy/menshealth/menshealth-supplements",
              "health-pharmacy/menshealth/jock-rash",
              "regaine/regaine-for-men",
              "mens/mens-toiletries/mens-hair-loss/hair-loss-concealers/nanogen-2",
              "health-pharmacy/lifestyle-wellbeing/weightloss/weight-management-shop-all",
              "health-pharmacy/electrical-health-diagnostics/blood-pressure-monitors",
              "health-pharmacy/electrical-health-diagnostics/back-neck-massagers",
              "health-pharmacy/electrical-health-diagnostics/activity-trackers-1",
              "health-pharmacy/electrical-health-diagnostics/foot-massagers-spas",
              "health-pharmacy/electrical-health-diagnostics/dna-test-kits",
              "health-pharmacy/electrical-health-diagnostics/thermometers",
              "health-pharmacy/electrical-health-diagnostics/electrical-footcare",
              "health-pharmacy/electrical-health-diagnostics/air-filters-humidifiers-de-humidifiers",
              "health-pharmacy/electrical-health-diagnostics/healthyheart",
              "health-pharmacy/electrical-health-diagnostics/light-therapy",
              "health-pharmacy/electrical-health-diagnostics/weighing-scales-body-fat-monitors",
              "health-pharmacy/electrical-health-diagnostics/electric-blankets",
              "health-pharmacy/livingaids/managing-medication",
              "health-pharmacy/livingaids/out-and-about",
              "health-pharmacy/livingaids/bathroom-toilet-aids",
              "health-pharmacy/livingaids/kitchen-home-aids",
              "health-pharmacy/livingaids/grab-rails",
              "health-pharmacy/travel-health/antimalarials",
              "health-pharmacy/travel-health/sunprotection",
              "health-pharmacy/travel-health/sunburn",
              "health-pharmacy/travel-health/insect-repellents",
              "health-pharmacy/travel-health/bites-stings",
              "health-pharmacy/travel-health/first-aid",
              "health-pharmacy/travel-health/travel-sickness",
              "health-pharmacy/travel-health/diarrhoea",
              "health-pharmacy/travel-health/ear-plugs",
              "health-pharmacy/travel-health/flight-socks",
              "health-pharmacy/travel-health/childrens-travel-health",
              "beauty/no7/no7-shop-all",
              "beauty/beauty-skincare-offers",
              "beauty/beauty-value-packs-and-bundles",
              "beauty/new-in-beauty-skincare",
              "beauty/trending-products",
              "beauty/luxury-beauty-skincare/luxury-beauty-gift",
              "beauty/luxury-beauty-skincare/luxury-beauty-offers",
              "beauty/luxury-beauty-skincare/all-luxury-skincare",
              "beauty/luxury-beauty-skincare/luxury-beauty-makeup",
              "beauty/luxury-beauty-skincare/luxury-value-packs-and-bundles",
              "beauty/luxury-beauty-skincare/luxury-makeup-tools",
              "beauty/luxury-beauty-skincare/luxury-beauty-men",
              "beauty/luxury-beauty-skincare/luxury-suncare-tanning-travel",
              "beauty/luxury-beauty-skincare/luxury-beauty-hair",
              "beauty/makeup/face",
              "beauty/makeup/eyes",
              "beauty/makeup/lips",
              "beauty/makeup/nails",
              "beauty/makeup/brushes-sponges",
              "beauty/makeup/glitter-accessories",
              "beauty/makeup/palettes",
              "beauty/makeup/make-up-remover-",
              "beauty/makeup/make-up-mirrors",
              "beauty/makeup/wash-bags",
              "beauty/skincare/skincare-all-skincare",
              "beauty/hair/all-hair",
              "beauty/beauty-accessories/accessories-shop-all",
              "beauty/travel-beauty-minis",
              "beauty/vegan-range",
              "fragrance/perfume/all-perfume",
              "fragrance/perfume/perfume-gift-sets",
              "fragrance/perfume/body-mists-",
              "fragrance/perfume/scented-bath-shower",
              "fragrance/aftershave/mens-aftershave",
              "fragrance/aftershave/aftershave-gift-sets",
              "fragrance/aftershave/cologne",
              "fragrance/aftershave/fragrance-bath-shower",
              "fragrance/vegan-fragrances",
              "fragrance/celebrity-fragrance ",
              "fragrance/fragrance-home-fragrances",
              "fragrance/fragrance-exclusives",
              "baby-child/baby-value-packs-and-bundles",
              "baby-child/pushchairs-car-seats/pushchairs-strollers-prams-doubles",
              "baby-child/pushchairs-car-seats/car-seats-accessories",
              "baby-child/pushchairs-car-seats/travel-systems",
              "baby-child/nursery-furniture/furniture-sets",
              "baby-child/nursery-furniture/furniture-sets",
              "baby-child/nursery-furniture/cots-cot-beds",
              "baby-child/nursery-furniture/cribs-moses-baskets",
              "baby-child/nursery-furniture/thermometers",
              "baby-child/nursery-furniture/baby-safety",
              "baby-child/nursery-furniture/bedside-sleeping",
              "baby-child/nursery-furniture/mobiles-night-lights",
              "baby-child/nursery-furniture/baby-monitors",
              "baby-child/nursery-furniture/baby-bedding",
              "baby-child/nursery-furniture/bouncers-swings-play-gyms",
              "baby-child/mothercare-clothing/shop-all-baby-kids-clothing",
              "baby-child/mothercare-clothing/new-clothing-collection",
              "baby-child/mothercare-clothing/mothercare-baby-clothes-0-24-months",
              "baby-child/mothercare-clothing/mothercare-girls-clothes-9-months-6-years",
              "baby-child/mothercare-clothing/mothercare-boys-clothes-9-months-6-years",
              "baby-child/mothercare-clothing/mothercare-nightwear-underwear",
              "baby-child/mothercare-clothing/mothercare-maternity-bras",
              "baby-child/mothercare-clothing/mothercare-premature-baby-range",
              "baby-child/babyfeeding/baby-value-packs-and-bundles",
              "baby-child/babyfeeding/baby-milk-formula",
              "baby-child/babyfeeding/baby-food-weaning",
              "baby-child/babyfeeding/breastfeeding-pumps",
              "baby-child/babyfeeding/toddler-food-drink",
              "baby-child/babyfeeding/baby-bottles-teats",
              "baby-child/babyfeeding/baby-cups",
              "baby-child/babyfeeding/soothers-teethers",
              "baby-child/babyfeeding/child-dinnerware",
              "baby-child/babyfeeding/sterilising",
              "baby-child/babyfeeding/bibs-muslins",
              "baby-child/babyfeeding/highchairs-booster-seats",
              "baby-child/bathing-changing/kids-dental",
              "baby-child/bathing-changing/changing-bag-essentials",
              "baby-child/bathing-changing/baby-value-packs-and-bundles",
              "baby-child/bathing-changing/baby-child-toiletries",
              "baby-child/bathing-changing/nappies",
              "baby-child/bathing-changing/baby-baths-accessories",
              "baby-child/bathing-changing/baby-wipes",
              "baby-child/bathing-changing/changing-bags-mats",
              "baby-child/bathing-changing/potty-training",
              "baby-child/bathing-changing/nappy-disposal",
              "baby-child/bathing-changing/baby-cotton-wool",
              "baby-child/pregnancy-maternity/hospital-bag-essentials",
              "baby-child/pregnancy-maternity/new-mum-toiletries",
              "baby-child/pregnancy-maternity/maternity-nursing-clothing",
              "baby-child/pregnancy-maternity/pregnancy-tests",
              "baby-child/pregnancy-maternity/pregnancysupplements",
              "baby-child/pregnancy-maternity/maternity-tens-machine",
              "baby-child/pregnancy-maternity/pregnancy-maternity-pillows",
              "baby-child/pregnancy-maternity/ovulation-fertility-tests",
              "baby-child/pregnancy-maternity/gift-baby-shower",
              "baby-child/baby-child-health/baby-child-vitamins",
              "baby-child/baby-child-health/fever-pain-relief",
              "baby-child/baby-child-health/child-cough-cold-flu",
              "baby-child/baby-child-health/skincare-conditions",
              "baby-child/baby-child-health/teething",
              "baby-child/baby-child-health/allergy-hayfever-children",
              "baby-child/baby-child-health/child-first-aid",
              "baby-child/baby-child-health/nits-lice-worms",
              "baby-child/baby-child-health/colic",
              "toys",
              "wellness/immunity-protection",
              "wellness/condoms-sexual-health/sexual-pleasure-shop-all",
              "wellness/weightloss/weight-management-shop-all",
              "wellness/sports-nutrition/sport-nutrition-shop-all",
              "wellness/wellness-supplements/wellness-supplements-shop-all",
              "wellness/digestive-health",
              "wellness/supplements-for-energy",
              "wellness/food-and-drink/wellness-shop-all",
              "wellness/food-and-drink/breakfast-bars-and-cereals-",
              "wellness/food-and-drink/drinks-and-teas",
              "wellness/food-and-drink/dry-fruits-nuts-and-seeds",
              "wellness/food-and-drink/free-from",
              "wellness/food-and-drink/honey-jam-and-spreads",
              "wellness/food-and-drink/savoury-snacks",
              "wellness/food-and-drink/wellness-superfoods",
              "wellness/food-and-drink/sweet-snacks",
              "wellness/food-and-drink/vegan-food",
              "wellness/sustainable-living/sustainable-bathroom-essentials",
              "wellness/sustainable-living/on-the-go",
              "wellness/sustainable-living/b-corp-brands",
              "wellness/beautysupplements",
              "wellness/activity-trackers-1",
              "wellness/recipe-book-accessories",
              "wellness/all-vegan-products",
              "toiletries/hair/all-hair",
              "toiletries/bootsdental/at-home-dentistry",
              "toiletries/bootsdental/mouthwash",
              "toiletries/bootsdental/kids-dental",
              "toiletries/bootsdental/oral-health",
              "toiletries/bootsdental/toothpaste",
              "toiletries/bootsdental/toothbrushes",
              "toiletries/bootsdental/teeth-whitening",
              "toiletries/bootsdental/flossing-accessories",
              "toiletries/bootsdental/denture-care",
              "electrical-dental/electric-toothbrushes",
              "electrical-dental/kids-electric-toothbrushes",
              "electrical/electrical-dental/dental-brush-heads",
              "electrical/electrical-dental/electric-flossers",
              "toiletries/washing-bathing/shower-gel",
              "toiletries/washing-bathing/bubble-bath-oil",
              "toiletries/washing-bathing/soap-hand-wash",
              "toiletries/washing-bathing/body-scrub",
              "toiletries/washing-bathing/face-wash",
              "toiletries/washing-bathing/tissues-wipes-sanitisers",
              "toiletries/washing-bathing/baby-child-toiletries",
              "toiletries/washing-bathing/toiletries-bath-sets",
              "toiletries/washing-bathing/natural-toiletries",
              "toiletries/washing-bathing/bath-accessories",
              "toiletries/washing-bathing/talcum-powder",
              "toiletries/washing-bathing/wash-bags",
              "toiletries/luxury-bath-body/bath-body-gifts-",
              "toiletries/mens-toiletries/shaving-grooming",
              "toiletries/mens-toiletries/mens-value-packs-and-bundles",
              "toiletries/mens-toiletries/mens-deodorants-antiperspirants",
              "toiletries/mens-toiletries/skincare-body",
              "toiletries/mens-toiletries/mens-washing-bathing",
              "toiletries/mens-toiletries/mens-hair",
              "toiletries/mens-toiletries/mens-gift-sets",
              "toiletries/feminine-hygiene/sanitary-towels",
              "toiletries/feminine-hygiene/feminine-wash-wipes",
              "toiletries/feminine-hygiene/tampons",
              "toiletries/feminine-hygiene/panty-liners",
              "toiletries/feminine-hygiene/female-incontinence",
              "toiletries/feminine-hygiene/menstrual-cups",
              "toiletries/feminine-hygiene/bladder-weakness",
              "toiletries/suncare/suncare-value-packs-bundles",
              "toiletries/suncare/sunprotection",
              "toiletries/suncare/kids-sun-protection",
              "toiletries/suncare/face-sun-protection",
              "toiletries/suncare/expert-skin-sun-protection",
              "toiletries/suncare/after-sun",
              "toiletries/fake-gradual-tan/fake-and-gradual-tan-all",
              "fake-gradual-tan/self-tan-prep-and-maintain",
              "fake-gradual-tan/self-tan-instant-tan",
              "fake-gradual-tan/self-tan-gradual-tan",
              "toiletries/deodorants-antiperspirants",
              "toiletries/skincare-female-hair-removal",
              "electrical/price-match-promise", #~ this is all electrical
              "mens/mens-toiletries/shaving-grooming",
              "mens/mens-toiletries/mens-value-packs-and-bundles",
              "mens/mens-toiletries/mens-deodorants-antiperspirants",
              "mens/mens-toiletries/skincare-body",
              "mens/mens-toiletries/mens-washing-bathing",
              "mens/mens-toiletries/mens-hair",
              "mens/mens-toiletries/aftershave",
              "mens/mens-toiletries/male-incontinence",
              "mens/mens-toiletries/mens-hair-loss",
              "mens/mens-toiletries/mens-gift-sets",
              "photo/photo-albums-frames/frames-photo",
              "photo/photo-albums-frames/photo-albums-range",
              "photo/headphones-cameras-accessories/cameras",
              "photo/headphones-cameras-accessories/headphones-earphones-speakers",
              "photo/headphones-cameras-accessories/batteries",
              "photo/headphones-cameras-accessories/phone-tablet-accessories",
              "photo/headphones-cameras-accessories/camera-accessories",
              "opticians/glasses/all-frames-boots-opticians",
              "opticians/prescription-sunglasses/all-sunglasses-boots-opticians",
              "holidays/travel-health/antimalarials",
              "holidays/travel-health/sunprotection",
              "holidays/travel-health/sunburn",
              "holidays/travel-health/insect-repellents",
              "holidays/travel-health/bites-stings",
              "holidays/travel-health/first-aid",
              "holidays/travel-health/travel-sickness",
              "holidays/travel-health/diarrhoea",
              "holidays/travel-health/ear-plugs",
              "holidays/travel-health/flight-socks",
              "holidays/travel-health/childrens-travel-health",
              "holidays/travel-essentials/luggage-tags",
              "holidays/travel-essentials/flight-socks",
              "holidays/travel-essentials/goggles",
              "holidays/travel-essentials/phone-tablet-accessories",
              "holidays/kids-travel",
              "holidays/sunglasses/all-sunglasses--1",
              "gift/her",
              "gift/him",
              "gift/candles-home-fragrance-for-her",
              "gift/gift-experience",
              "gift/birthday-gifts",
              "wellness/weightloss/weight-management-shop-all",]


uncategorised = ["https://www.boots.com/ellaone-ulipristal-acetate-tablet-30mg-10193530",
                 "https://www.boots.com/levonelle-one-step-1500-microgram-tablet-10083266",
                 "https://www.boots.com/levonorgestrel-emergency-contraception-10261514",
                 "https://www.boots.com/regaine-for-women-once-a-day-scalp-foam-5-w-w-cutaneous-foam-minoxidil-extra-strength---4-months-supply-2-x-73ml-10257670p",
                 "https://www.boots.com/nourkrin-woman-180s-3-month-supply-10102277",
                 "https://www.boots.com/nourkrin-woman-60-tablets-1-month-supply-10150737",
                 "https://www.boots.com/nourkrin-man-60-tablets-1-month-supply-10150740",
                 "https://www.boots.com/nourkrin-woman-360s-6-months-supply-10150766",
                 "https://www.boots.com/nourkrin-woman-6-months-free-2x-nourkrin-shampoo-and-scalp-cleanser-150ml-and-2x-nourkrin-conditioner-150ml-10171331",
                 "https://www.boots.com/nourkrin-woman-12-months-free-4x-nourkrin-shampoo-and-scalp-cleanser-150ml-and-4x-nourkrin-conditioner-150ml-10171352",
                 "https://www.boots.com/nourkrin-man-6-months-free-2x-nourkrin-shampoo-and-scalp-cleanser-150ml-and-2x-nourkrin-conditioner-150ml-10171355",
                 "https://www.boots.com/hairburst-volume-growth-elixir-120ml-10257652",
                 "https://www.boots.com/hairburst-3-month-supply-180-capsules-10208997",
                 "https://www.boots.com/wellness/vitaminsandsupplements/vitamins-supplements-shop-by-ingredient/selenium/Hairburst-Strawberry-Chewable-Vitamins-60-Gummies-10223512",
                 "https://www.boots.com/hairburst-1-month-supply-60-capsules-10202117",
                 "https://www.boots.com/wellness/beautysupplements/hairburst-unicorn-vegan-hair-vitamins---60-pastilles-10268002",
                 "https://www.boots.com/hairburst-womens-35plus-hair-vitamins---60-capsules-10268025",
                 "https://www.boots.com/hairburst-healthy-hair-for-new-mums-30-capsules-10270019",
                 "https://www.boots.com/hairburst-chewable-hair-vitamins-15-day-supply-10267954",
                 "https://www.boots.com/viviscal-womens-max-strength-supplement-30-tablets-10180654",
                 "https://www.boots.com/viviscal-womens-max-strength-60s-1-month-supply-10154692",
                 "https://www.boots.com/viviscal-womens-max-strength-supplements-90-tablets-10220342",
                 "https://www.boots.com/beauty/hair/hair-health-vitamins/viviscal-mans-supplements-760s-tablets-12-month-supply",
                 "https://www.boots.com/beauty/hair/hair-health-vitamins/viviscal-mans-supplements-360s-tablets-6-month-supply",
                 "https://www.boots.com/viviscal-womens-max-strength-supplements-bundle-360s-6months-supply",
                 "https://www.boots.com/viviscal-womens-max-strength-supplements-180s-3months-supply-10154727",
                 "https://www.boots.com/toppik-hair-building-fibres-medium-brown-12g-10180699",
                 "https://www.boots.com/toppik-hair-building-fibres-black-12g-10180694",]


fields_to_extract = [["single", "div", "class", "productid"],
                     ["single", "h1", "itemprop", "name"],
                     ["single", "div", "id", "PDP_productPrice"],
                     ["single", "div", "class", "details"],
                     ["multi", "div", "data-slot-id", "14"],  #~ full product box, usually
                     ["multi", "div", "data-slot-id", "13"],] #~ full product box, sometimes
