搜索客，搜索人自己的社区

社区日报第1608期 (2023-04-10)

社区日报 • yuebancanghai 发表了文章 • 0 个评论 • 4189 次浏览 • 2023-04-10 08:59 • 来自相关话题

Web Scraper + Elasticsearch + Kibana + SearchKit 打造的豆瓣电影top250 搜索演示系统

Elasticsearch • 森发表了文章 • 0 个评论 • 6143 次浏览 • 2023-04-09 10:56 • 来自相关话题

Web Scraper + Elasticsearch + Kibana + SearchKit 打造的豆瓣电影top250 搜索演示系统

作者：小森同学

声明：电影数据来源于“豆瓣电影”，如有侵权，请联系删除

Web Scraper

json { "_id": "top250", "startUrl": ["<a href="https://movie.douban.com/top250?start=" rel="nofollow" target="_blank">https://movie.douban.com/top250?start=</a>[0-225:25]&filter="], "selectors": [{ "id": "container", "multiple": true, "parentSelectors": ["_root"], "selector": ".grid_view li", "type": "SelectorElement" }, { "id": "name", "multiple": false, "parentSelectors": ["container"], "regex": "", "selector": "span.title:nth-of-type(1)", "type": "SelectorText" }, { "id": "number", "multiple": false, "parentSelectors": ["container"], "regex": "", "selector": "em", "type": "SelectorText" }, { "id": "score", "multiple": false, "parentSelectors": ["container"], "regex": "", "selector": "span.rating_num", "type": "SelectorText" }, { "id": "review", "multiple": false, "parentSelectors": ["container"], "regex": "", "selector": "span.inq", "type": "SelectorText" }, { "id": "year", "multiple": false, "parentSelectors": ["container"], "regex": "\\d{4}", "selector": "p:nth-of-type(1)", "type": "SelectorText" }, { "id": "tour_guide", "multiple": false, "parentSelectors": ["container"], "regex": "^导演: \\S*", "selector": "p:nth-of-type(1)", "type": "SelectorText" }, { "id": "type", "multiple": false, "parentSelectors": ["container"], "regex": "[^/]+$", "selector": "p:nth-of-type(1)", "type": "SelectorText" }, { "id": "area", "multiple": false, "parentSelectors": ["container"], "regex": "[^\\/]+(?=\\/[^\\/]*$)", "selector": "p:nth-of-type(1)", "type": "SelectorText" }, { "id": "detail_link", "multiple": false, "parentSelectors": ["container"], "selector": ".hd a", "type": "SelectorLink" }, { "id": "director", "multiple": false, "parentSelectors": ["detail_link"], "regex": "", "selector": "span:nth-of-type(1) .attrs a", "type": "SelectorText" }, { "id": "screenwriter", "multiple": false, "parentSelectors": ["detail_link"], "regex": "(?<=编剧: )[\\u4e00-\\u9fa5A-Za-z0-9/()\\·\\s]+(?=主演)", "selector": "div#info", "type": "SelectorText" }, { "id": "film_length", "multiple": false, "parentSelectors": ["detail_link"], "regex": "\\d+", "selector": "span[property='v:runtime']", "type": "SelectorText" }, { "id": "IMDb", "multiple": false, "parentSelectors": ["detail_link"], "regex": "(?<=[IMDb:\\s+])\\S*(?=\\d*$)", "selector": "div#info", "type": "SelectorText" }, { "id": "language", "multiple": false, "parentSelectors": ["detail_link"], "regex": "(?<=语言: )\\S+", "selector": "div#info", "type": "SelectorText" }, { "id": "alias", "multiple": false, "parentSelectors": ["detail_link"], "regex": "(?<=又名: )[\\u4e00-\\u9fa5A-Za-z0-9/()\\s]+(?=IMDb)", "selector": "div#info", "type": "SelectorText" }, { "id": "pic", "multiple": false, "parentSelectors": ["container"], "selector": "img", "type": "SelectorImage" }] } 

elasticsearch

 { "mappings": { "properties": { "IMDb": { "type": "keyword", "copy_to": [ "all" ] }, "alias": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "all": { "type": "text", "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "area": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "director": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "film_length": { "type": "long" }, "id": { "type": "keyword" }, "language": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "link": { "type": "keyword" }, "name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "number": { "type": "long" }, "photo": { "type": "keyword" }, "review": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "score": { "type": "double" }, "screenwriter": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "type": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } }, "copy_to": [ "all" ], "analyzer": "ik_max_word", "search_analyzer": "ik_smart" }, "year": { "type": "long" } } } } 

kibana

需要使用pipeline对索引字段进行处理，如对type 通过空格进行分割为数组等，可以参照官方文档或其他博客。

制作仪表板省略, 请自行搜索

SearchKit

参考 https://github.com/searchkit/searchkit-starter-app