Extraction et filtrage des URLs sur la pagee d'accuiel

@config(age=10 * 24 * 60 * 60)
def page_accueil(self, response):
    for element in response.doc('a[href^="http"]').items():
        if re.match('http://www.yunjinet.com/sell/show.+', element.attr.href):
            self.crawl(element.attr.href, callback=self.page_detail)
        else:
            self.crawl(element.attr.href, callback=self.page_accueil)

Parcours des URLs sur la page d'accueil

    @config(age=10 * 24 * 60 * 60)
    def page_accueil(self, response):
        # Extraction des URLs de produits
        for element in response.doc('.list_products_row_box .list_products_name > a').items():
            self.crawl(element.attr.href, callback=self.page_detail, validate_cert=False)
        
        # Navigation vers la page suivante
        suivant = response.doc('nav a').attr.href
        self.crawl(suivant, callback=self.page_accueil, validate_cert=False)

Pagination incrémentale classique

from pyspider.libs.base_handler import *
 
 
class Collecteur(BaseHandler):
    crawl_config = {
    }
    
    def __init__(self):
        self.url_base = 'https://mm.taobao.com/json/request_top_list.htm?page='
        self.numero_page = 1
        self.total_pages = 30
 
    @every(minutes=24 * 60)
    def on_start(self):
        while self.numero_page <= self.total_pages:
            url = self.url_base + str(self.numero_page)
            print(url)
            self.crawl(url, callback=self.page_accueil)
            self.numero_page += 1
 
    @config(age=10 * 24 * 60 * 60)
    def page_accueil(self, response):
        for element in response.doc('a[href^="http"]').items():
            self.crawl(element.attr.href, callback=self.page_detail)
 
    @config(priority=2)
    def page_detail(self, response):
        return {
            "url": response.url,
            "title": response.doc('title').text(),
        }

Parcours des URLs sur la page de détail

    @config(priority=2)
    def page_detail(self, response):
        # Extraction des images de la page courante
        for element in response.doc('.xzoom-container > img.img-responsive').items():
            url_image = element.attr.src
        
        # Navigation vers la page suivante
        for element in response.doc('.pagenavi > a:last-child').items():
            self.crawl(element.attr.href, callback=self.page_detail, validate_cert=False)

Stockgae des données dans MySQL

from pyspider.libs.base_handler import *
import re
from pyspider.database.mysql.mysqldb import SQL
import html

# Configuration d'accès à la base de données

config_bdd = {
'hote': '127.0.0.1',
'utilisateur': 'root',
'mot_de_passe': 'root',
'base_donnees': 'pyspider',
}

#Création de la table de stockage

@every(minutes=24 * 60)
def on_start(self):
self.creer_table_resultats()



# Création de la table [résultats du scraper]
def on_result(self, resultat):
if not resultat or not resultat['v_metatags_title_1']:
return
sql = SQL(hote=config_bdd['hote'], utilisateur=config_bdd['utilisateur'],
mot_de_passe=config_bdd['mot_de_passe'], base_donnees=config_bdd['base_donnees'])
sql.insert(self.nom_projet + '_resultats', **resultat)

# Méthode de création de la table [résultats]
def creer_table_resultats(self):
sql = SQL(hote=config_bdd['hote'], utilisateur=config_bdd['utilisateur'],
mot_de_passe=config_bdd['mot_de_passe'], base_donnees=config_bdd['base_donnees'])

requete_creation = """CREATE TABLE IF NOT EXISTS `""" + self.nom_projet + '_resultats' + """` (
`v_products_model` mediumtext NOT NULL,
`v_products_type` int(2) DEFAULT '1',
`v_products_image` mediumtext NOT NULL,
`v_products_name_1` mediumtext NOT NULL,
`v_products_description_1` mediumtext NOT NULL,
`v_products_url_1` mediumtext NOT NULL,
`v_specials_price` int(2) DEFAULT '0',
`v_specials_date_avail` varchar(50) DEFAULT '0000-00-00 00:00:00',
`v_specials_expires_date` varchar(50) DEFAULT '0000-00-00 00:00:00',
`v_products_price` mediumtext NOT NULL,
`v_products_weight` int(2) DEFAULT '0',
`v_product_is_call` int(2) DEFAULT '0',
`v_products_sort_order` int(2) DEFAULT '1',
`v_products_quantity_order_min` int(2) DEFAULT '1',
`v_products_quantity_order_units` int(2) DEFAULT '1',
`v_products_priced_by_attribute` int(2) DEFAULT '0',
`v_product_is_always_free_shipping` int(2) DEFAULT '1',
`v_date_avail` varchar(50) DEFAULT '0000-00-00 00:00:00',
`v_date_added` varchar(50) DEFAULT '0000-00-00 00:00:00',
`v_products_quantity` int(10) DEFAULT '1000',
`v_manufacturers_name` mediumtext NOT NULL,
`v_categories_name_1` mediumtext NOT NULL,
`v_tax_class_title` varchar(50) DEFAULT '--none--',
`v_status` int(2) DEFAULT '1',
`v_html_uri` mediumtext NOT NULL,
`v_products_options_type` int(2) DEFAULT '0',
`v_metatags_title_1` mediumtext NOT NULL,
`v_metatags_keywords_1` mediumtext NOT NULL,
`v_metatags_description_1` mediumtext NOT NULL,
`v_products_options_name_1` mediumtext NOT NULL,
`v_products_options_values_name_1` mediumtext NOT NULL,
`v_products_options_name_2` mediumtext NOT NULL,
`v_products_options_values_name_2` mediumtext NOT NULL
) ENGINE=MyISAM DEFAULT CHARSET=utf8"""
try:
resultat_sql = sql.execute(requete_creation)
sql.disconnect()
return resultat_sql is not False
except Exception as erreur:
print("[Table de résultats] Erreur détectée: " + str(erreur))
return False