抓取百度明星资料

public class BaiduYuLeStar {/** * 保存出错 */private static StringBuilder noResult = new StringBuilder();/** * 当前页数 */private static int pagePos = 1;public static void main(String[] args) {BaiduYuLeStar star = new BaiduYuLeStar();star.work(Integer.MAX_VALUE);// star.workOnePage();}/** * 影视娱乐资料库的基地址 */private final String baseDataURL = "http://data.yule.baidu.com";/** * 页面编码 */private final String charset = "gbk";/** * 捉取页面工具类 */private HtmlCleaner cleaner;/** * star表的dao类实例 */private final StarDao dao = StarDao.getInstance();/** * 日志记录 */private final Logger logger = LoggerFactory.getLogger(this.getClass());/** * 之后定时运行捉取新增加的明星，以 一个阀值为准 */private final int pageNum = 1;/** * 获取所有明星的URL地址 */private final String starURL = "http://data.yule.baidu.com/index_star_s0_p#.html";/** * 构造函数，先调用初始化的函数 */public BaiduYuLeStar() {initCleaner();}/** * 获取当前页面的结点 * * @return */private TagNode getPage() {TagNode node = getPage(pagePos);pagePos++;// 下一页return node;}/** * 获取指定某一页面的结点 * * @param pageNum * @return */private TagNode getPage(int pageNum) {String pageURL = starURL.replace("#", String.valueOf(pageNum));// 测试// pageURL="http://data.yule.baidu.com/index_star_s0_p4.html";try {TagNode pageNode = cleaner.clean(new URL(pageURL), charset);TagNode node = HtmlCleanerXpathUtil.xpathOne(pageNode,"//*[@id='container']");return node;} catch (IOException e) {logger.info(e.getMessage(), e);}return null;}/** * 从详细的页面获取相关的详细信息并封装到类Star * * @param star * 相对地址 * @return */public void getStarFullInfo(Star star) {try {URL url = new URL(star.getFromURL());TagNode starInfoNode = cleaner.clean(url, charset);// //*[@id='photo']/div/div/h2TagNode nameNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"//*[@id='photo']//h2");if (nameNode == null) {logger.info("没有获取到名称");noResult.append(star.getFromURL() + " :没有获取到名称/n");return;}// 保存名字String name = nameNode.getText().toString().trim();star.setName(StringUtils.unicodeTextDecode(name).toString());// 保存图片TagNode photoNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"//*[@id='photo']/div/div/img");if (photoNode == null) {photoNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"//*[@id='photo']/img");}String imgURL = photoNode.getAttributeByName("src");star.setPoster(imgURL);// 保存id，放在获取图片之后，因为图片的名称跟id可能相关TagNode idNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"/head");handleID(star, idNode.getText().toString().trim());// 其它基本信息handleBaseInfo(star, starInfoNode);// 获取并保存简介信息TagNode introNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"//div[@class='intro']");String summary = introNode.getText().toString().trim();star.setIntro(summary);} catch (IOException e) {logger.info(e.getMessage(), e);}}/** * 获取当前页面的所有明星的信息的链接 * * @return */public List<Star> getStarInfoHref() {List<Star> starHrefList = new ArrayList<Star>();TagNode pageNode = getPage();if (pageNode == null) {return starHrefList;}TagNode starNode[] = HtmlCleanerXpathUtil.xpathArray(pageNode,"//li//a");for (TagNode node : starNode) {String href = node.getAttributeByName("href");if (href.startsWith("/star_details/")) {//找到详细资料的链接Star star = new Star();star.setFromURL(href);starHrefList.add(star);}}return starHrefList;}/** * 处理基本信息的节点 * * @param star * 封装明星的值对象 * @param starInfoNode */private void handleBaseInfo(Star star, TagNode starInfoNode) {TagNode detailNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"//div[@class='infolink infohome']");if (detailNode == null) {// 多个版本的页面结构不一样，比如：吉赛贝-托纳多雷detailNode = HtmlCleanerXpathUtil.xpathOne(starInfoNode,"//div[@class='infolink']");}if (detailNode == null) {logger.info("detailNode为空，没有获取到基本信息的节点");noResult.append(star.getName() + " 没有获取到基本信息的节点/n");return;}String baseInfoStr[] = detailNode.getText().toString().trim().split("(/n)|( )");for (String map : baseInfoStr) {if (!map.trim().equals("")) {String temp[] = map.split("：");if (temp.length != 2) {continue;}String mapName = temp[0].trim();String mapValue = temp[1].trim();if (mapName.equals("姓名")) {star.setAlias(mapValue);} else if (mapName.equals("性别")) {int sex = 0;if ("男".equals(star.getGender())) {sex = 1;} else if ("女".equals(star.getGender())) {sex = 2;} else if ("组合".equals(star.getGender())) {sex = 3;}star.setGender(sex);} else if (mapName.equals("身高")) {// 处理身高int height = 0;if (mapValue != null) {String hStr = mapValue;String regex = "([0-9]+)";Pattern pattern = Pattern.compile(regex);Matcher matcher = pattern.matcher(hStr);if (matcher.find()) {height = Integer.valueOf(matcher.group(1));}}star.setHeight(height);} else if (mapName.equals("地区")) {star.setZone(mapValue);} else if (mapName.equals("星座")) {star.setConstellation(mapValue);} else if (mapName.equals("血型")) {star.setBloodType(mapValue);} else if (mapName.equals("职业")) {star.setJob(mapValue);}}}}/** * 从页面的某个内容获取id号 * * @param idText * @return */private void handleID(Star star, String idText) {String regex = "ProductID//s*=//s*([0-9]+)";Pattern p = Pattern.compile(regex);Matcher m = p.matcher(idText);if (m.find()) {star.setId(Integer.valueOf(m.group(1)));} else {// 从图片名称获取if (star.getPoster() != null) {int nameIndex = star.getPoster().lastIndexOf("/");String name = star.getPoster().substring(nameIndex + 1,star.getPoster().length());String id = name.replace(".jpg", "");star.setId(Integer.valueOf(id));}}}/** * 处理获得的明星对象 * * @param star */private void handleStar(Star star) {System.out.println(star);boolean isSucSave = dao.saveStar(star);logger.info("保存到数据库 " + isSucSave);}/** * 初始化HtmlCleaner实例，在构造函数里调用 */private void initCleaner() {cleaner = new HtmlCleaner();CleanerProperties props = cleaner.getProperties();props.setUseEmptyElementTags(false);// props.setRecognizeUnicodeChars(true);// props.set}/** * 初次捉取全部页面 *//** * * @param numPage * 要捉取的页面数量 */public void work(int numPage) {int num = 1;int index = 0;do {List<Star> starList = getStarInfoHref();if (starList == null || starList.size() <= 0) {logger.info("结束捉取。。");break;}System.out.println(starList.size());for (Star star : starList) {index = star.getFromURL().lastIndexOf("/");String first = star.getFromURL().substring(0, index + 1);String last = star.getFromURL().substring(index + 1,star.getFromURL().length());try {last = URLEncoder.encode(last, "gbk");last = last.replaceAll("//+", "%20");last = last.replaceAll("///", "%2");String href = first + last;star.setFromURL(baseDataURL + href);} catch (UnsupportedEncodingException e) {logger.info(e.getMessage(), e);}getStarFullInfo(star);handleStar(star);// break;}num++;} while (num <= numPage);System.out.println(noResult.toString());}/** * 捉取一个页面 */public void workOnePage() {work(pageNum);}}