User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Allow: /wp-includes/js/
Allow: /wp-includes/css/
Disallow: /wp-content/backupwordpress-a1454e5489-backups/
Disallow: /wp-content/cache/
Disallow: /wp-content/envato-backups/
Disallow: /wp-content/images/
Disallow: /wp-content/languages/
Disallow: /wp-content/upgrade/
Disallow: /readme.html
Disallow: /*.txt
Allow: *.css
Allow: *.js
Sitemap: https://www.crm-buzzee.fr/sitemap_index.xml