无法使用 puppeteer 获取现有对象内的 URL [英] unable to get URL inside existing object using puppeteer

问题描述

以下是我的 HTML :

<源源srcset="https://assets.myntassets.com/f_webp,dpr_1.0,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg ,https://assets.myntassets.com/f_webp,dpr_1.5,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 1.5x,https://assets.myntassets.com/f_webp,dpr_1.8,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 1.8x,https://assets.myntassets.com/f_webp,dpr_2.0,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 2.0x,https://assets.myntassets.com/f_webp,dpr_2.2,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 2.2x,https://assets.myntassets.com/f_webp,dpr_2.4,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 2.4x,https://assets.myntassets.com/f_webp,dpr_2.6,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 2.6x,https://assets.myntassets.com/f_webp,dpr_2.8,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-oundBlack-Printed-RT-shirt-2881525433792598-1.jpg 2.8x"type=image/webp"><img src="https://assets.myntassets.com/dpr_2,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525465379-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg"class="img-responsive";alt=HERE&amp;NOW 男士黑色印花圆领 T 恤";title=HERE&amp;NOW 男士黑色印花圆领 T 恤";样式=宽度:100%；显示:块;></图片></div></div></div></div><div类=product-productMetaInfo"><h3类=product-brand">HERE&amp;NOW</h3><h4 class=product-product">男士印花圆领 T 恤</h4><h4 class="product-sizes"><!-- react-text: 396 -->Sizes: <!--/react-text --><span class="product-sizeInventoryPresent">S, <;/span><span class="product-sizeInventoryPresent">M, </span><span class="product-sizeInventoryPresent">L, </span><span class=";product-sizeInventoryPresent">XL，</span><span class=product-sizeInventoryPresent">XXL</span></h4><div class=product-price"><span><span class="product-discountedPrice"><!-- react-text: 405 -->Rs.<!--/react-text --><!-- react-text: 406 -->374<!--/react-text --></span><span class=产品罢工"><!--反应文本:408->Rs.<!--/react-text --><!-- react-text: 409 -->749<!--/react-text --></span></span><span class="product-discountPercentage">(50% OFF)</span></div></div></a><div class="image-grid-similarColorsCtaproduct-similarItemCta"><span class="myntraweb-sprite image-grid-similarColorsIcon sprites-similarProductsIcon"</span><span class="image-grid-iconText">VIEW SIMILAR</span></div><div class="product-actions><span class="product-actionsButton product-wishlist"样式=宽度:100%；text-align: center;"><!-- react-text: 416 -->wishlist<!--/react-text --></span></div><divclass="product-sizeDisplayDiv"><div class="product-sizeDisplayHeader"><span>选择尺寸</span><span class="myntraweb-sprite product-sizeDisplayRemoveMark sprites-remove";></span></div><div class="product-sizeButtonsContaier"><button class="product-sizeButton">S</button><button class=";product-sizeButton">M</button><button class=product-sizeButton">L</button><button class=product-sizeButton">XL</button>button class=product-sizeButton">XXL</button></div></div>";

当前代码:

const res = await page.evaluate(() => {const productArry = [...document.querySelectorAll(.product-base")];返回 productArry.map((product) => {让 productSizeText = product.querySelector(".product-sizes").innerText;让 productSizeArr = productSizeText.replace("尺寸:", "").修剪().split(",");返回 {imageurl: product.querySelector("div>图片>.img-responsive").src,品牌名称:product.querySelector(.product-brand").innerText,产品名称:product.querySelector(.product-product").innerText,productSizes: productSizeArr,};});});

是不是由于延迟加载我在从上面的标签中获取 src 时出现空错误

解决方案

请尝试以下代码片段:

//获取所有源标签url让 imageURLArr = 等待 page.evaluate(() => {//这将获得DOM的第一个sourceTag，如果它有更多的源标签并且不是第一个源标签元素，则根据您要废弃的DOM更改值0让 sourceTag = document.getElementsByTagName('source')[0];//检查选择器是否存在如果(源标签){//这将为您提供源标签的所有图像 URL让 imagURLs = sourceTag.getAttribute('srcset')返回 imageURL；}});控制台日志(imageURLArr)；//要获得产品品牌名称，您可以这样做等待 page.waitForSelector('h3');const BrandName = await page.evaluate(() => document.getElementsByClassName('product-brand').textContent);console.log('品牌名称 = ' + 品牌名称);//要获得产品尺寸，您可以这样做让 productSizes = await page.$$eval('.product-sizeInventoryPresent', elements => {让大小 = elements.map((element) => element.textContent);返回大小；});

following is my HTML :

<div class="product-thumbShim"></div><a target="_blank" href="tshirts/herenow/herenow-men-black-printed-round-neck-t-shirt/4318138/buy" style="display: block;"><div class="product-imageSliderContainer"><div class="product-sliderContainer" style="display: block;"><div style="background: rgb(244, 255, 249);"><div style="height: 280px; width: 100%;"><picture class="img-responsive" style="width: 100%; height: 100%; display: block;"><source srcset="
    https://assets.myntassets.com/f_webp,dpr_1.0,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg ,
    https://assets.myntassets.com/f_webp,dpr_1.5,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 1.5x,
    https://assets.myntassets.com/f_webp,dpr_1.8,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 1.8x,
    https://assets.myntassets.com/f_webp,dpr_2.0,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 2.0x,
    https://assets.myntassets.com/f_webp,dpr_2.2,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 2.2x,
    https://assets.myntassets.com/f_webp,dpr_2.4,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 2.4x,
    https://assets.myntassets.com/f_webp,dpr_2.6,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 2.6x,
    https://assets.myntassets.com/f_webp,dpr_2.8,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg 2.8x" type="image/webp"><img src="https://assets.myntassets.com/dpr_2,q_60,w_210,c_limit,fl_progressive/assets/images/4318138/2018/5/4/11525433792765-HERENOW-Men-Black-Printed-Round-Neck-T-shirt-2881525433792598-1.jpg" class="img-responsive" alt="HERE&amp;NOW Men Black Printed Round Neck T-shirt" title="HERE&amp;NOW Men Black Printed Round Neck T-shirt" style="width: 100%; display: block;"></picture></div></div></div></div><div class="product-productMetaInfo"><h3 class="product-brand">HERE&amp;NOW</h3><h4 class="product-product">Men Printed Round Neck T-shirt</h4><h4 class="product-sizes"><!-- react-text: 396 -->Sizes: <!-- /react-text --><span class="product-sizeInventoryPresent">S, </span><span class="product-sizeInventoryPresent">M, </span><span class="product-sizeInventoryPresent">L, </span><span class="product-sizeInventoryPresent">XL, </span><span class="product-sizeInventoryPresent">XXL</span></h4><div class="product-price"><span><span class="product-discountedPrice"><!-- react-text: 405 -->Rs. <!-- /react-text --><!-- react-text: 406 -->374<!-- /react-text --></span><span class="product-strike"><!-- react-text: 408 -->Rs. <!-- /react-text --><!-- react-text: 409 -->749<!-- /react-text --></span></span><span class="product-discountPercentage">(50% OFF)</span></div></div></a><div class="image-grid-similarColorsCta product-similarItemCta"><span class="myntraweb-sprite image-grid-similarColorsIcon sprites-similarProductsIcon"></span><span class="image-grid-iconText">VIEW SIMILAR</span></div><div class="product-actions "><span class="product-actionsButton product-wishlist " style="width: 100%; text-align: center;"><!-- react-text: 416 -->wishlist<!-- /react-text --></span></div><div class="product-sizeDisplayDiv"><div class="product-sizeDisplayHeader"><span>Select a size</span><span class="myntraweb-sprite product-sizeDisplayRemoveMark sprites-remove"></span></div><div class="product-sizeButtonsContaier"><button class="product-sizeButton">S</button><button class="product-sizeButton">M</button><button class="product-sizeButton">L</button><button class="product-sizeButton">XL</button><button class="product-sizeButton">XXL</button></div></div>"

current code :

const res = await page.evaluate(() => {
        const productArry = [...document.querySelectorAll(".product-base")];

        return productArry.map((product) => {
            let productSizeText = product.querySelector(".product-sizes").innerText;
            let productSizeArr = productSizeText
                .replace("Sizes:", "")
                .trim()
                .split(",");

            return {
                imageurl: product.querySelector("div > picture > .img-responsive")
                    .src,
                brandName: product.querySelector(".product-brand").innerText,
                productName: product.querySelector(".product-product").innerText,
                productSizes: productSizeArr,
            };
        });
    });

is it due to lazy-loading I am getting null error while getting src from the above tag

解决方案

Please try the below code snippets:

//To get all the source tag urls
let imageURLArr = await page.evaluate(() => {
    //This will get the first sourceTag of the DOM, change the value 0 according to your DOM that you are scrapping if it has more source tags and is not the first source tag element
    let sourceTag = document.getElementsByTagName('source')[0];
    // check selector exists
    if (sourceTag) {
        // This will give you all the image URLs of source tag
        let imagURLs = sourceTag.getAttribute('srcset')
        return imagURLs;
    }
});

console.log(imageURLArr);

//To get the product brand name you can do this
await page.waitForSelector('h3');
const brandName = await page.evaluate(() => document.getElementsByClassName('product-brand').textContent);
console.log('Brand Name = ' + brandName);

// To get the product Sizes you can do this
let productSizes = await page.$$eval('.product-sizeInventoryPresent', elements => {
    let sizes = elements.map((element) => element.textContent);
    return sizes;
});

这篇关于无法使用 puppeteer 获取现有对象内的 URL的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！