用Java抓取谷歌图片

Web scraping Google Images with Java

本文关键字:谷歌 抓取 Java      更新时间:2023-09-26

我的任务是制作一个应用程序,用户可以使用它来搜索他们Query的三张图片。它可以用任何语言编写,有人建议使用Python。然而,我对Python的GUI技能很差,我最熟悉Java,并且我已经下载了Eclipse SWT包,我打算使用它。

好的,到目前为止你做了什么

我介绍了他们制作一个简单网络浏览器的示例代码。我让它显示谷歌图片作为主页,并打算使用JavaScript/jQuery来自动填写他们的查询,并返回前三张图片。(我还计划以某种方式使用JavaScript/jQuery来返回这三张图片。)

根据Eclipse SWT的文档,evaluate()execute()用于评估、执行JavaScript:http://help.eclipse.org/indigo/index.jsp?topic=/org.eclipse.platform.doc.isv/reference/api/org/eclipse/swt/browser/Browser.html

这是我使用过的代码:

import org.eclipse.swt.*;
import org.eclipse.swt.browser.Browser;
import org.eclipse.swt.browser.LocationEvent;
import org.eclipse.swt.browser.LocationListener;
import org.eclipse.swt.browser.ProgressEvent;
import org.eclipse.swt.browser.ProgressListener;
import org.eclipse.swt.browser.StatusTextEvent;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.ProgressBar;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Text;
import org.eclipse.swt.widgets.ToolBar;
import org.eclipse.swt.widgets.ToolItem;
public class BrowserCodeDemo {
    public static void main(String[] args) {
        Display display = new Display();
        final Shell shell = new Shell(display);
        GridLayout gridLayout = new GridLayout();
        gridLayout.numColumns = 3;
        shell.setLayout(gridLayout);
        ToolBar toolbar = new ToolBar(shell, SWT.NONE);
        ToolItem itemBack = new ToolItem(toolbar, SWT.PUSH);
        itemBack.setText("Back");
        ToolItem itemForward = new ToolItem(toolbar, SWT.PUSH);
        itemForward.setText("Forward");
        ToolItem itemStop = new ToolItem(toolbar, SWT.PUSH);
        itemStop.setText("Stop");
        ToolItem itemRefresh = new ToolItem(toolbar, SWT.PUSH);
        itemRefresh.setText("Refresh");
        ToolItem itemGo = new ToolItem(toolbar, SWT.PUSH);
        itemGo.setText("Go");
        GridData data = new GridData();
        data.horizontalSpan = 3;
        toolbar.setLayoutData(data);
        Label labelAddress = new Label(shell, SWT.NONE);
        labelAddress.setText("Address");
        final Text location = new Text(shell, SWT.BORDER);
        data = new GridData();
        data.horizontalAlignment = GridData.FILL;
        data.horizontalSpan = 2;
        data.grabExcessHorizontalSpace = true;
        location.setLayoutData(data);
        final Browser browser;
        try {
            browser = new Browser(shell, SWT.NONE);
        } catch (SWTError e) {
            System.out.println("Could not instantiate Browser: " + e.getMessage());
            display.dispose();
            return;
        }
        data = new GridData();
        data.horizontalAlignment = GridData.FILL;
        data.verticalAlignment = GridData.FILL;
        data.horizontalSpan = 3;
        data.grabExcessHorizontalSpace = true;
        data.grabExcessVerticalSpace = true;
        browser.setLayoutData(data);
        final Label status = new Label(shell, SWT.NONE);
        data = new GridData(GridData.FILL_HORIZONTAL);
        data.horizontalSpan = 2;
        status.setLayoutData(data);
        final ProgressBar progressBar = new ProgressBar(shell, SWT.NONE);
        data = new GridData();
        data.horizontalAlignment = GridData.END;
        progressBar.setLayoutData(data);
        /* event handling */
        Listener listener = new Listener() {
            @Override
            public void handleEvent(Event event) {
                ToolItem item = (ToolItem)event.widget;
                String string = item.getText();
                if (string.equals("Back")) browser.back(); 
                else if (string.equals("Forward")) browser.forward();
                else if (string.equals("Stop")) browser.stop();
                else if (string.equals("Refresh")) browser.refresh();
                else if (string.equals("Go")) browser.setUrl(location.getText());
           }
        };
        browser.addProgressListener(new ProgressListener() {
            @Override
            public void changed(ProgressEvent event) {
                    if (event.total == 0) return;                            
                    int ratio = event.current * 100 / event.total;
                    progressBar.setSelection(ratio);
            }
            @Override
            public void completed(ProgressEvent event) {
                progressBar.setSelection(0);
            }
        });
        browser.addLocationListener(new LocationListener() {
            @Override
            public void changed(LocationEvent event) {
                if (event.top) location.setText(event.location);
            }
            @Override
            public void changing(LocationEvent event) 
            {
                //System.out.println("Something is happening.");
            }
        });
        itemBack.addListener(SWT.Selection, listener);
        itemForward.addListener(SWT.Selection, listener);
        itemStop.addListener(SWT.Selection, listener);
        itemRefresh.addListener(SWT.Selection, listener);
        itemGo.addListener(SWT.Selection, listener);
        location.addListener(SWT.DefaultSelection, new Listener() {
            @Override
            public void handleEvent(Event e) 
            {
                browser.setUrl(location.getText());
                System.out.println("New URL loaded");
            }
        });
        shell.open();
        browser.setUrl(
            "https://www.google.com/imghp?hl=en&tab=wi&ei=m8g4VLndMaz4igKlvoDADg&ved=0CAMQqi4oAQ");
        //browser.setVisible(false);
        boolean jQueryExecuted = browser.execute("$(''#lst-ib'').val(''snopes'')");
        if (!jQueryExecuted)
        {
            System.out.println("Your jQuery didn't execute.");
        }
        jQueryExecuted = browser.execute("$(''[name=btnG]'').click()");
        if (!jQueryExecuted)
        {
            System.out.println("Your jQuery didn't execute.");
        }
        while (!shell.isDisposed()) {
            if (!display.readAndDispatch())
                display.sleep();
        }
        display.dispose();
    }
}

我不认为问题出在代码本身,因为我试图自动设置文本字段,但在Java和浏览器中都失败了;/*文本字段的名称为"lst-ib"*/

你想在网上刮什么

我正在尝试在网上抓取谷歌图片:https://www.google.com/imghp?hl=en&tab=wi&ei=m8g4VLndMaz4igKlvoDADg&ved=0CAMQqi4oAQ。

文本字段是名称为"lst-ib"的输入。该按钮是一个名为"btnG"的输入。文本字段位于id为"gs_lc0"的div中。(我也看到了多个其他大小完全相同的输入元素)。

哦,哇。10个小时过去了,除了我,没有人对此发表评论。事实证明,问题比我想象的要严重得多:Eclipse SWT中的浏览器无法执行jQuery。

幸运的是,我找到了Google Image API,它是用JavaScript编写的,不推荐使用,但仍然有效。此外,我在页面上发现了一些简单的源代码:https://developers.google.com/image-search/v1/devguide#hiworld

我将newImg.src = "/image-search/v1/result.tbUrl;"更改为newImg.src = result.tbUrl;,以便加载图像。这改变了一切。


你的新计划是什么

我现在计划使用Java中的这段代码。我计划将src更改为JavaScript代码使用的Google Image API的src,然后调用该代码来获得前三个图像。显然,这就是从Java运行JavaScript的方式:如何在Java编程中运行JavaScript。如果我能想出如何将Java使用的JavaScript引擎的src更改为:https://www.google.com/jsapi