add category_limit, store_limit, product_limit

Author: HannaAA17

README.md

@@ -1,22 +1,32 @@
-```bash
-scrapy crawl 1_stores -a store="https://shop.despar.com"
-```
+## How to run
+1) **Stores Spyder** scrapes stores info in the domain [[shop.despar.com](https://shop.despar.com), [shop.desparsicilia.it](https://shop.desparsicilia.it)]
 
-``` bash
-scrapy crawl 2_product_list -a dev=True
-```
+    ```bash
+    scrapy crawl 1_stores -a store="https://shop.despar.com"
+    ```
 
-```bash
-scrapy crawl 3_product_details -a dev=True
-```
+2) **Product List Spyder** scrapes the categories, products-list on each category and the promos in this store
+
+    ``` bash
+    scrapy crawl 2_product_list -a store_limit=1 -a category_limit=3 
+    ```
+
+3) **Product Details Spyder** scrapes the product descriptions and images links 
+    ```bash
+    scrapy crawl 3_product_details -a product_limit=3
+    ```
 
 ---
 
+## Endpoints
+
+```https://{domain}/spesa-consegna-domicilio/{zip_code}```
+
+```https://{domain}/spesa-ritiro-negozio/{store_slug}```
+
+```{store_url}/{category_slug}```
 
-https://{ domain }/spesa-consegna-domicilio/{ zip_code }
-https://{ domain }/spesa-ritiro-negozio/{ store_slug }
+```{store_url}/prodotto/{product_id}```
 
-{ store_url }/{ category_slug }
-{ store_url }/prodotto/{ product_id }
-{ store_url }/ajax/productsPagination?{ category_id, page_num }
+```{store_url}/ajax/productsPagination?{category_id,page_num}```
 

despar_scraper/spiders/2_product_list.py

@@ -35,15 +35,16 @@ class ProductListSpider(scrapy.Spider):
         }
     }
 
-    def __init__(self, store_list_file='data/json/stores.json', dev=None, *args, **kwargs):
+    def __init__(self, store_list_file='data/json/stores.json', store_limit='0', category_limit='0', *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.store_list_file = store_list_file
-        self.dev = dev
+        self.category_limit = int(category_limit)
+        self.store_limit = int(store_limit)
 
     async def start(self):
         # open json file in async function
         with open(self.store_list_file, 'r') as f:
-            store_list = json.load(f) if not self.dev else json.load(f)[:1]
+            store_list = json.load(f) if not self.store_limit else json.load(f)[:self.store_limit]
 
         for store in store_list:
             yield scrapy.Request(
@@ -54,7 +55,7 @@ async def start(self):
 
     def parse__get_categories(self, response: HtmlResponse):
         store = response.meta['store']
-
+        cat_count = 0
         # Extract categories
         for main_tag in response.css('div.main-navigation__item--container'):
             main_category = main_tag.css('div.main-navigation__item--title > span::text').get(default='').strip()
@@ -105,9 +106,11 @@ def parse__get_categories(self, response: HtmlResponse):
                             'page': 1, 'attempt': 1
                         },
                     )
-                
-                    if self.dev:
-                        return None
+
+                    cat_count += 1
+                    if self.category_limit:
+                        if cat_count >= self.category_limit:
+                            return None
 
 
     def parse__get_products_list(self, response: HtmlResponse):

despar_scraper/spiders/3_product_details.py

@@ -21,15 +21,15 @@ class ProductDetailsSpider(scrapy.Spider):
         }
     }
 
-    def __init__(self, product_list_file='data/json/product_list.json', dev=False, *args, **kwargs):
+    def __init__(self, product_list_file='data/json/product_list.json', product_limit='0', *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.product_list_file = product_list_file
-        self.dev = dev
+        self.product_limit = int(product_limit)
 
     async def start(self):
         # open json file in async function
         with open(self.product_list_file, 'r') as f:
-            product_list = json.load(f) if not self.dev else json.load(f)[:10]
+            product_list = json.load(f) if not self.product_limit else json.load(f)[:self.product_limit]
 
         for prod in product_list:
             yield scrapy.Request(