YuJian
/
nodeSpider_sexy
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

添加新网站

main
YuJian 3 years ago
parent
ae9fb256c2
commit
3dcdc6f5b3
4 changed files with 225 additions and 443 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .gitignore
  2. 555
      package-lock.json
  3. 10
      utils/saveImages.js
  4. 102
      xinwenba.js

1
.gitignore vendored
Unescape View File

@ -41,6 +41,7 @@ build/Release
node_modules/ node_modules/
jspm_packages/ jspm_packages/
Result/ Result/
Cos/
# TypeScript v1 declaration files # TypeScript v1 declaration files
typings/ typings/

555
package-lock.json generated
Unescape View File

@ -2,45 +2,6 @@
"requires": true, "requires": true,
"lockfileVersion": 1, "lockfileVersion": 1,
"dependencies": { "dependencies": {
"ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmmirror.com/ajv/download/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"asn1": {
"version": "0.2.6",
"resolved": "https://registry.npmmirror.com/asn1/download/asn1-0.2.6.tgz?cache=0&sync_timestamp=1635986760581&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fasn1%2Fdownload%2Fasn1-0.2.6.tgz",
"integrity": "sha1-DTp7tuZOAqkMAwOzHykoaOoJoI0=",
"requires": {
"safer-buffer": "~2.1.0"
}
},
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.nlark.com/assert-plus/download/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npm.taobao.org/asynckit/download/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npm.taobao.org/aws-sign2/download/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
},
"aws4": {
"version": "1.11.0",
"resolved": "https://registry.nlark.com/aws4/download/aws4-1.11.0.tgz",
"integrity": "sha1-1h9G2DslGSUOJ4Ta9bCUeai0HFk="
},
"axios": { "axios": {
"version": "0.24.0", "version": "0.24.0",
"resolved": "https://registry.npmmirror.com/axios/download/axios-0.24.0.tgz", "resolved": "https://registry.npmmirror.com/axios/download/axios-0.24.0.tgz",
@ -49,19 +10,6 @@
"follow-redirects": "^1.14.4" "follow-redirects": "^1.14.4"
} }
}, },
"bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npm.taobao.org/bcrypt-pbkdf/download/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=",
"requires": {
"tweetnacl": "^0.14.3"
}
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npm.taobao.org/caseless/download/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"cheerio": { "cheerio": {
"version": "1.0.0-rc.10", "version": "1.0.0-rc.10",
"resolved": "https://registry.npmmirror.com/cheerio/download/cheerio-1.0.0-rc.10.tgz", "resolved": "https://registry.npmmirror.com/cheerio/download/cheerio-1.0.0-rc.10.tgz",
@ -74,24 +22,17 @@
"parse5": "^6.0.1", "parse5": "^6.0.1",
"parse5-htmlparser2-tree-adapter": "^6.0.1", "parse5-htmlparser2-tree-adapter": "^6.0.1",
"tslib": "^2.2.0" "tslib": "^2.2.0"
}
},
"cheerio-select": {
"version": "1.5.0",
"resolved": "https://registry.nlark.com/cheerio-select/download/cheerio-select-1.5.0.tgz",
"integrity": "sha1-+vPa6zGxfF4anavO4oiq+Kr6WCM=",
"requires": {
"css-select": "^4.1.3",
"css-what": "^5.0.1",
"domelementtype": "^2.2.0",
"domhandler": "^4.2.0",
"domutils": "^2.7.0"
}, },
"dependencies": { "dependencies": {
"css-what": { "dom-serializer": {
"version": "5.1.0", "version": "1.3.2",
"resolved": "https://registry.npmmirror.com/css-what/download/css-what-5.1.0.tgz", "resolved": "https://registry.nlark.com/dom-serializer/download/dom-serializer-1.3.2.tgz?cache=0&sync_timestamp=1621256830355&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdom-serializer%2Fdownload%2Fdom-serializer-1.3.2.tgz",
"integrity": "sha1-P3tweq32M7r2LCzrhXm1RbtA9/4=" "integrity": "sha1-YgZDfTLO767HFhgDIwx6ILwbTZE=",
"requires": {
"domelementtype": "^2.0.1",
"domhandler": "^4.2.0",
"entities": "^2.0.0"
}
}, },
"domelementtype": { "domelementtype": {
"version": "2.2.0", "version": "2.2.0",
@ -105,111 +46,63 @@
"requires": { "requires": {
"domelementtype": "^2.2.0" "domelementtype": "^2.2.0"
} }
}
}
},
"combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.nlark.com/combined-stream/download/combined-stream-1.0.8.tgz",
"integrity": "sha1-w9RaizT9cwYxoRCoolIGgrMdWn8=",
"requires": {
"delayed-stream": "~1.0.0"
}
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.nlark.com/core-util-is/download/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"css-select": {
"version": "4.2.1",
"resolved": "https://registry.npmmirror.com/css-select/download/css-select-4.2.1.tgz",
"integrity": "sha512-/aUslKhzkTNCQUB2qTX84lVmfia9NyjP3WpDGtj/WxhwBzWBYUV3DgUpurHTme8UTPcPlAD1DJ+b0nN/t50zDQ==",
"requires": {
"boolbase": "^1.0.0",
"css-what": "^5.1.0",
"domhandler": "^4.3.0",
"domutils": "^2.8.0",
"nth-check": "^2.0.1"
},
"dependencies": {
"boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npm.taobao.org/boolbase/download/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
}, },
"css-what": { "domutils": {
"version": "5.1.0", "version": "2.8.0",
"resolved": "https://registry.npmmirror.com/css-what/download/css-what-5.1.0.tgz", "resolved": "https://registry.nlark.com/domutils/download/domutils-2.8.0.tgz?cache=0&sync_timestamp=1630106606599&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdomutils%2Fdownload%2Fdomutils-2.8.0.tgz",
"integrity": "sha1-P3tweq32M7r2LCzrhXm1RbtA9/4=" "integrity": "sha1-RDfe9dtuLR9dbuhZvZXKfQIEgTU=",
"requires": {
"dom-serializer": "^1.0.1",
"domelementtype": "^2.2.0",
"domhandler": "^4.2.0"
}
}, },
"domelementtype": { "entities": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.npm.taobao.org/domelementtype/download/domelementtype-2.2.0.tgz?cache=0&sync_timestamp=1617298545989&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdomelementtype%2Fdownload%2Fdomelementtype-2.2.0.tgz", "resolved": "https://registry.nlark.com/entities/download/entities-2.2.0.tgz?cache=0&sync_timestamp=1628508189267&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fentities%2Fdownload%2Fentities-2.2.0.tgz",
"integrity": "sha1-mgtsJ4LtahxzI9QiZxg9+b2LHVc=" "integrity": "sha1-CY3JDruD2N/6CJ1VJWs1HTTE2lU="
}, },
"domhandler": { "htmlparser2": {
"version": "4.3.0", "version": "6.1.0",
"resolved": "https://registry.npmmirror.com/domhandler/download/domhandler-4.3.0.tgz", "resolved": "https://registry.npmmirror.com/htmlparser2/download/htmlparser2-6.1.0.tgz?cache=0&sync_timestamp=1636640940074&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fhtmlparser2%2Fdownload%2Fhtmlparser2-6.1.0.tgz",
"integrity": "sha512-fC0aXNQXqKSFTr2wDNZDhsEYjCiYsDWl3D01kwt25hm1YIPyDGHvvi3rw+PLqHAl/m71MaiF7d5zvBr0p5UB2g==", "integrity": "sha1-xNditsM3GgXb5l6UrkOp+EX7j7c=",
"requires": { "requires": {
"domelementtype": "^2.2.0" "domelementtype": "^2.0.1",
"domhandler": "^4.0.0",
"domutils": "^2.5.2",
"entities": "^2.0.0"
} }
}, },
"nth-check": { "parse5": {
"version": "2.0.1", "version": "6.0.1",
"resolved": "https://registry.nlark.com/nth-check/download/nth-check-2.0.1.tgz", "resolved": "https://registry.npm.taobao.org/parse5/download/parse5-6.0.1.tgz",
"integrity": "sha1-Lv4WL1w9oGoolZ+9PbddvuqfD8I=", "integrity": "sha1-4aHAhcVps9wIMhGE8Zo5zCf3wws="
},
"parse5-htmlparser2-tree-adapter": {
"version": "6.0.1",
"resolved": "https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/download/parse5-htmlparser2-tree-adapter-6.0.1.tgz",
"integrity": "sha1-LN+a2CMyEUA3DU2/XT6Sx8jdxuY=",
"requires": { "requires": {
"boolbase": "^1.0.0" "parse5": "^6.0.1"
} }
},
"tslib": {
"version": "2.3.1",
"resolved": "https://registry.npmmirror.com/tslib/download/tslib-2.3.1.tgz",
"integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw=="
} }
} }
}, },
"dashdash": { "cheerio-select": {
"version": "1.14.1", "version": "1.5.0",
"resolved": "https://registry.npm.taobao.org/dashdash/download/dashdash-1.14.1.tgz?cache=0&sync_timestamp=1601073454623&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdashdash%2Fdownload%2Fdashdash-1.14.1.tgz", "resolved": "https://registry.nlark.com/cheerio-select/download/cheerio-select-1.5.0.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", "integrity": "sha1-+vPa6zGxfF4anavO4oiq+Kr6WCM=",
"requires": {
"assert-plus": "^1.0.0"
}
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npm.taobao.org/delayed-stream/download/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"dom-serializer": {
"version": "1.3.2",
"resolved": "https://registry.nlark.com/dom-serializer/download/dom-serializer-1.3.2.tgz?cache=0&sync_timestamp=1621256830355&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdom-serializer%2Fdownload%2Fdom-serializer-1.3.2.tgz",
"integrity": "sha1-YgZDfTLO767HFhgDIwx6ILwbTZE=",
"requires": {
"domelementtype": "^2.0.1",
"domhandler": "^4.2.0",
"entities": "^2.0.0"
}
},
"domelementtype": {
"version": "2.2.0",
"resolved": "https://registry.npm.taobao.org/domelementtype/download/domelementtype-2.2.0.tgz?cache=0&sync_timestamp=1617298545989&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdomelementtype%2Fdownload%2Fdomelementtype-2.2.0.tgz",
"integrity": "sha1-mgtsJ4LtahxzI9QiZxg9+b2LHVc="
},
"domhandler": {
"version": "4.3.0",
"resolved": "https://registry.npmmirror.com/domhandler/download/domhandler-4.3.0.tgz",
"integrity": "sha512-fC0aXNQXqKSFTr2wDNZDhsEYjCiYsDWl3D01kwt25hm1YIPyDGHvvi3rw+PLqHAl/m71MaiF7d5zvBr0p5UB2g==",
"requires": {
"domelementtype": "^2.2.0"
}
},
"domutils": {
"version": "2.8.0",
"resolved": "https://registry.nlark.com/domutils/download/domutils-2.8.0.tgz?cache=0&sync_timestamp=1630106606599&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdomutils%2Fdownload%2Fdomutils-2.8.0.tgz",
"integrity": "sha1-RDfe9dtuLR9dbuhZvZXKfQIEgTU=",
"requires": { "requires": {
"dom-serializer": "^1.0.1", "css-select": "^4.1.3",
"css-what": "^5.0.1",
"domelementtype": "^2.2.0", "domelementtype": "^2.2.0",
"domhandler": "^4.2.0" "domhandler": "^4.2.0",
"domutils": "^2.7.0"
}, },
"dependencies": { "dependencies": {
"dom-serializer": { "dom-serializer": {
@ -235,6 +128,16 @@
"domelementtype": "^2.2.0" "domelementtype": "^2.2.0"
} }
}, },
"domutils": {
"version": "2.8.0",
"resolved": "https://registry.nlark.com/domutils/download/domutils-2.8.0.tgz?cache=0&sync_timestamp=1630106606599&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdomutils%2Fdownload%2Fdomutils-2.8.0.tgz",
"integrity": "sha1-RDfe9dtuLR9dbuhZvZXKfQIEgTU=",
"requires": {
"dom-serializer": "^1.0.1",
"domelementtype": "^2.2.0",
"domhandler": "^4.2.0"
}
},
"entities": { "entities": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.nlark.com/entities/download/entities-2.2.0.tgz?cache=0&sync_timestamp=1628508189267&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fentities%2Fdownload%2Fentities-2.2.0.tgz", "resolved": "https://registry.nlark.com/entities/download/entities-2.2.0.tgz?cache=0&sync_timestamp=1628508189267&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fentities%2Fdownload%2Fentities-2.2.0.tgz",
@ -242,93 +145,33 @@
} }
} }
}, },
"ecc-jsbn": { "css-select": {
"version": "0.1.2", "version": "4.2.1",
"resolved": "https://registry.npm.taobao.org/ecc-jsbn/download/ecc-jsbn-0.1.2.tgz", "resolved": "https://registry.npmmirror.com/css-select/download/css-select-4.2.1.tgz",
"integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", "integrity": "sha512-/aUslKhzkTNCQUB2qTX84lVmfia9NyjP3WpDGtj/WxhwBzWBYUV3DgUpurHTme8UTPcPlAD1DJ+b0nN/t50zDQ==",
"requires": {
"jsbn": "~0.1.0",
"safer-buffer": "^2.1.0"
}
},
"entities": {
"version": "2.2.0",
"resolved": "https://registry.nlark.com/entities/download/entities-2.2.0.tgz?cache=0&sync_timestamp=1628508189267&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fentities%2Fdownload%2Fentities-2.2.0.tgz",
"integrity": "sha1-CY3JDruD2N/6CJ1VJWs1HTTE2lU="
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.nlark.com/extend/download/extend-3.0.2.tgz",
"integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo="
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmmirror.com/extsprintf/download/extsprintf-1.3.0.tgz?cache=0&sync_timestamp=1635889740043&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fextsprintf%2Fdownload%2Fextsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
},
"fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npm.taobao.org/fast-deep-equal/download/fast-deep-equal-3.1.3.tgz",
"integrity": "sha1-On1WtVnWy8PrUSMlJE5hmmXGxSU="
},
"fast-json-stable-stringify": {
"version": "2.1.0",
"resolved": "https://registry.npm.taobao.org/fast-json-stable-stringify/download/fast-json-stable-stringify-2.1.0.tgz",
"integrity": "sha1-h0v2nG9ATCtdmcSBNBOZ/VWJJjM="
},
"follow-redirects": {
"version": "1.14.6",
"resolved": "https://registry.npmmirror.com/follow-redirects/download/follow-redirects-1.14.6.tgz",
"integrity": "sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A=="
},
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npm.taobao.org/forever-agent/download/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
},
"form-data": {
"version": "2.3.3",
"resolved": "https://registry.npm.taobao.org/form-data/download/form-data-2.3.3.tgz",
"integrity": "sha1-3M5SwF9kTymManq5Nr1yTO/786Y=",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.6",
"mime-types": "^2.1.12"
}
},
"getpass": {
"version": "0.1.7",
"resolved": "https://registry.nlark.com/getpass/download/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"requires": {
"assert-plus": "^1.0.0"
}
},
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.nlark.com/har-schema/download/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
},
"har-validator": {
"version": "5.1.5",
"resolved": "https://registry.npmmirror.com/har-validator/download/har-validator-5.1.5.tgz",
"integrity": "sha1-HwgDufjLIMD6E4It8ezds2veHv0=",
"requires": {
"ajv": "^6.12.3",
"har-schema": "^2.0.0"
}
},
"htmlparser2": {
"version": "6.1.0",
"resolved": "https://registry.npmmirror.com/htmlparser2/download/htmlparser2-6.1.0.tgz?cache=0&sync_timestamp=1636640940074&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fhtmlparser2%2Fdownload%2Fhtmlparser2-6.1.0.tgz",
"integrity": "sha1-xNditsM3GgXb5l6UrkOp+EX7j7c=",
"requires": { "requires": {
"domelementtype": "^2.0.1", "boolbase": "^1.0.0",
"domhandler": "^4.0.0", "css-what": "^5.1.0",
"domutils": "^2.5.2", "domhandler": "^4.3.0",
"entities": "^2.0.0" "domutils": "^2.8.0",
"nth-check": "^2.0.1"
}, },
"dependencies": { "dependencies": {
"boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npm.taobao.org/boolbase/download/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
},
"dom-serializer": {
"version": "1.3.2",
"resolved": "https://registry.nlark.com/dom-serializer/download/dom-serializer-1.3.2.tgz?cache=0&sync_timestamp=1621256830355&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdom-serializer%2Fdownload%2Fdom-serializer-1.3.2.tgz",
"integrity": "sha1-YgZDfTLO767HFhgDIwx6ILwbTZE=",
"requires": {
"domelementtype": "^2.0.1",
"domhandler": "^4.2.0",
"entities": "^2.0.0"
}
},
"domelementtype": { "domelementtype": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.npm.taobao.org/domelementtype/download/domelementtype-2.2.0.tgz?cache=0&sync_timestamp=1617298545989&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdomelementtype%2Fdownload%2Fdomelementtype-2.2.0.tgz", "resolved": "https://registry.npm.taobao.org/domelementtype/download/domelementtype-2.2.0.tgz?cache=0&sync_timestamp=1617298545989&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdomelementtype%2Fdownload%2Fdomelementtype-2.2.0.tgz",
@ -342,217 +185,53 @@
"domelementtype": "^2.2.0" "domelementtype": "^2.2.0"
} }
}, },
"domutils": {
"version": "2.8.0",
"resolved": "https://registry.nlark.com/domutils/download/domutils-2.8.0.tgz?cache=0&sync_timestamp=1630106606599&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fdomutils%2Fdownload%2Fdomutils-2.8.0.tgz",
"integrity": "sha1-RDfe9dtuLR9dbuhZvZXKfQIEgTU=",
"requires": {
"dom-serializer": "^1.0.1",
"domelementtype": "^2.2.0",
"domhandler": "^4.2.0"
}
},
"entities": { "entities": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.nlark.com/entities/download/entities-2.2.0.tgz?cache=0&sync_timestamp=1628508189267&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fentities%2Fdownload%2Fentities-2.2.0.tgz", "resolved": "https://registry.nlark.com/entities/download/entities-2.2.0.tgz?cache=0&sync_timestamp=1628508189267&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fentities%2Fdownload%2Fentities-2.2.0.tgz",
"integrity": "sha1-CY3JDruD2N/6CJ1VJWs1HTTE2lU=" "integrity": "sha1-CY3JDruD2N/6CJ1VJWs1HTTE2lU="
},
"nth-check": {
"version": "2.0.1",
"resolved": "https://registry.nlark.com/nth-check/download/nth-check-2.0.1.tgz",
"integrity": "sha1-Lv4WL1w9oGoolZ+9PbddvuqfD8I=",
"requires": {
"boolbase": "^1.0.0"
}
} }
} }
}, },
"http-signature": { "css-what": {
"version": "1.2.0", "version": "5.1.0",
"resolved": "https://registry.npmmirror.com/http-signature/download/http-signature-1.2.0.tgz?cache=0&sync_timestamp=1637178987671&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fhttp-signature%2Fdownload%2Fhttp-signature-1.2.0.tgz", "resolved": "https://registry.npmmirror.com/css-what/download/css-what-5.1.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", "integrity": "sha1-P3tweq32M7r2LCzrhXm1RbtA9/4="
"requires": {
"assert-plus": "^1.0.0",
"jsprim": "^1.2.2",
"sshpk": "^1.7.0"
}
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npm.taobao.org/is-typedarray/download/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
},
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.nlark.com/isstream/download/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.nlark.com/jsbn/download/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
},
"json-schema": {
"version": "0.4.0",
"resolved": "https://registry.npmmirror.com/json-schema/download/json-schema-0.4.0.tgz?cache=0&sync_timestamp=1636423494254&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fjson-schema%2Fdownload%2Fjson-schema-0.4.0.tgz",
"integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.nlark.com/json-schema-traverse/download/json-schema-traverse-0.4.1.tgz",
"integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA="
},
"json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npm.taobao.org/json-stringify-safe/download/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
},
"jsprim": {
"version": "1.4.2",
"resolved": "https://registry.npmmirror.com/jsprim/download/jsprim-1.4.2.tgz",
"integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==",
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
"json-schema": "0.4.0",
"verror": "1.10.0"
}
},
"mime-db": {
"version": "1.51.0",
"resolved": "https://registry.npmmirror.com/mime-db/download/mime-db-1.51.0.tgz?cache=0&sync_timestamp=1636425951030&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fmime-db%2Fdownload%2Fmime-db-1.51.0.tgz",
"integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g=="
},
"mime-types": {
"version": "2.1.34",
"resolved": "https://registry.npmmirror.com/mime-types/download/mime-types-2.1.34.tgz?cache=0&sync_timestamp=1636432355847&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fmime-types%2Fdownload%2Fmime-types-2.1.34.tgz",
"integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==",
"requires": {
"mime-db": "1.51.0"
}
},
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.nlark.com/oauth-sign/download/oauth-sign-0.9.0.tgz",
"integrity": "sha1-R6ewFrqmi1+g7PPe4IqFxnmsZFU="
},
"parse5": {
"version": "6.0.1",
"resolved": "https://registry.npm.taobao.org/parse5/download/parse5-6.0.1.tgz",
"integrity": "sha1-4aHAhcVps9wIMhGE8Zo5zCf3wws="
},
"parse5-htmlparser2-tree-adapter": {
"version": "6.0.1",
"resolved": "https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/download/parse5-htmlparser2-tree-adapter-6.0.1.tgz",
"integrity": "sha1-LN+a2CMyEUA3DU2/XT6Sx8jdxuY=",
"requires": {
"parse5": "^6.0.1"
}
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.nlark.com/performance-now/download/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
},
"psl": {
"version": "1.8.0",
"resolved": "https://registry.npm.taobao.org/psl/download/psl-1.8.0.tgz",
"integrity": "sha1-kyb4vPsBOtzABf3/BWrM4CDlHCQ="
}, },
"punycode": { "follow-redirects": {
"version": "2.1.1", "version": "1.14.7",
"resolved": "https://registry.nlark.com/punycode/download/punycode-2.1.1.tgz?cache=0&sync_timestamp=1622604519710&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fpunycode%2Fdownload%2Fpunycode-2.1.1.tgz", "resolved": "https://registry.npmmirror.com/follow-redirects/download/follow-redirects-1.14.7.tgz",
"integrity": "sha1-tYsBCsQMIsVldhbI0sLALHv0eew=" "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ=="
},
"qs": {
"version": "6.5.2",
"resolved": "https://registry.npmmirror.com/qs/download/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
}, },
"request": { "iconv-lite": {
"version": "2.88.2", "version": "0.6.3",
"resolved": "https://registry.npmmirror.com/request/download/request-2.88.2.tgz", "resolved": "https://registry.nlark.com/iconv-lite/download/iconv-lite-0.6.3.tgz",
"integrity": "sha1-1zyRhzHLWofaBH4gcjQUb2ZNErM=", "integrity": "sha1-pS+AvzjaGVLrXGgXkHGYcaGnJQE=",
"requires": { "requires": {
"aws-sign2": "~0.7.0", "safer-buffer": ">= 2.1.2 < 3.0.0"
"aws4": "^1.8.0",
"caseless": "~0.12.0",
"combined-stream": "~1.0.6",
"extend": "~3.0.2",
"forever-agent": "~0.6.1",
"form-data": "~2.3.2",
"har-validator": "~5.1.3",
"http-signature": "~1.2.0",
"is-typedarray": "~1.0.0",
"isstream": "~0.1.2",
"json-stringify-safe": "~5.0.1",
"mime-types": "~2.1.19",
"oauth-sign": "~0.9.0",
"performance-now": "^2.1.0",
"qs": "~6.5.2",
"safe-buffer": "^5.1.2",
"tough-cookie": "~2.5.0",
"tunnel-agent": "^0.6.0",
"uuid": "^3.3.2"
} }
}, },
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.nlark.com/safe-buffer/download/safe-buffer-5.2.1.tgz",
"integrity": "sha1-Hq+fqb2x/dTsdfWPnNtOa3gn7sY="
},
"safer-buffer": { "safer-buffer": {
"version": "2.1.2", "version": "2.1.2",
"resolved": "https://registry.nlark.com/safer-buffer/download/safer-buffer-2.1.2.tgz", "resolved": "https://registry.nlark.com/safer-buffer/download/safer-buffer-2.1.2.tgz",
"integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo="
},
"sshpk": {
"version": "1.16.1",
"resolved": "https://registry.npm.taobao.org/sshpk/download/sshpk-1.16.1.tgz",
"integrity": "sha1-+2YcC+8ps520B2nuOfpwCT1vaHc=",
"requires": {
"asn1": "~0.2.3",
"assert-plus": "^1.0.0",
"bcrypt-pbkdf": "^1.0.0",
"dashdash": "^1.12.0",
"ecc-jsbn": "~0.1.1",
"getpass": "^0.1.1",
"jsbn": "~0.1.0",
"safer-buffer": "^2.0.2",
"tweetnacl": "~0.14.0"
}
},
"tough-cookie": {
"version": "2.5.0",
"resolved": "https://registry.npm.taobao.org/tough-cookie/download/tough-cookie-2.5.0.tgz",
"integrity": "sha1-zZ+yoKodWhK0c72fuW+j3P9lreI=",
"requires": {
"psl": "^1.1.28",
"punycode": "^2.1.1"
}
},
"tslib": {
"version": "2.3.1",
"resolved": "https://registry.npmmirror.com/tslib/download/tslib-2.3.1.tgz",
"integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw=="
},
"tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.nlark.com/tunnel-agent/download/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"requires": {
"safe-buffer": "^5.0.1"
}
},
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npm.taobao.org/tweetnacl/download/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
},
"uri-js": {
"version": "4.4.1",
"resolved": "https://registry.npm.taobao.org/uri-js/download/uri-js-4.4.1.tgz?cache=0&sync_timestamp=1610237624359&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Furi-js%2Fdownload%2Furi-js-4.4.1.tgz",
"integrity": "sha1-mxpSWVIlhZ5V9mnZKPiMbFfyp34=",
"requires": {
"punycode": "^2.1.0"
}
},
"uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmmirror.com/uuid/download/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A=="
},
"verror": {
"version": "1.10.0",
"resolved": "https://registry.npmmirror.com/verror/download/verror-1.10.0.tgz?cache=0&sync_timestamp=1635885078723&other_urls=https%3A%2F%2Fregistry.npmmirror.com%2Fverror%2Fdownload%2Fverror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"requires": {
"assert-plus": "^1.0.0",
"core-util-is": "1.0.2",
"extsprintf": "^1.2.0"
}
} }
} }
} }

10
utils/saveImages.js
Unescape View File

@ -5,15 +5,15 @@ const fs = require("fs");
* 图片下载 * 图片下载
* @param {*} imgObject 图片 URL * @param {*} imgObject 图片 URL
*/ */
function saveImages(imgObject, total) { function saveImages(imgObject, total, path = "Result") {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const forItem = Object.keys(imgObject); const forItem = Object.keys(imgObject);
for (let forItemIndex = 0; forItemIndex < forItem.length; forItemIndex++) { for (let forItemIndex = 0; forItemIndex < forItem.length; forItemIndex++) {
const imTitle = forItem[forItemIndex]; // 文件名 const imTitle = forItem[forItemIndex]; // 文件名
fs.access(`./Result/${imTitle}`, async (accessErr) => { fs.access(`./${path}/${imTitle}`, async (accessErr) => {
if (accessErr) fs.mkdirSync(`./Result/${imTitle}`); if (accessErr) fs.mkdirSync(`./${path}/${imTitle}`);
else { else {
const alreadyPath = fs.readdirSync(`./Result/${imTitle}`); const alreadyPath = fs.readdirSync(`./${path}/${imTitle}`);
// 已有文件跳过 // 已有文件跳过
if (alreadyPath.length === total) { if (alreadyPath.length === total) {
console.log(`${imTitle} 已存在 跳过抓取`); console.log(`${imTitle} 已存在 跳过抓取`);
@ -31,7 +31,7 @@ function saveImages(imgObject, total) {
url: eachItem, url: eachItem,
responseType: "arraybuffer", responseType: "arraybuffer",
}); });
fs.writeFileSync(`./Result/${imTitle}/${filename}`, data, "binary"); fs.writeFileSync(`./${path}/${imTitle}/${filename}`, data, "binary");
} catch (error) { } catch (error) {
console.log(`saveImages: 下载图片时出现错误！`); console.log(`saveImages: 下载图片时出现错误！`);
console.log(error); console.log(error);

102
xinwenba.js
Unescape View File

@ -0,0 +1,102 @@
const axios = require("axios");
const cheerio = require("cheerio");
const iconv = require("iconv-lite");
const saveImages = require("./utils/saveImages");
const request = axios.create({
methods: "GET",
timeout: 20 * 1000,
headers: {
"content-type": "text/html; charset=utf-8",
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
pragma: "no-cache",
referer: "https://www.xinwenba.net/web/meinv/",
"sec-ch-ua":
'" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "Windows",
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "same-origin",
"sec-fetch-user": "?1",
"upgrade-insecure-requests": 1,
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36",
},
});
let CURRY_PAGENUMBER = 1; // 爬取起始页码
let MAX_PAGENUMBER = 15; // 爬取最大页码
const spiderQueue = async (soureUrl) => {
const url = `${soureUrl.slice(0, 41)}${CURRY_PAGENUMBER}.html`;
request({ url }).then(async ({ data }) => {
const $ = cheerio.load(data);
const requestQueue = [];
$(".list_image a[class]").each((i, elem) => {
requestQueue.push({
title: $(elem).html(),
url: `https://www.xinwenba.net${$(elem).attr("href")}`,
});
});
for (let index = 0; index < requestQueue.length; index++) {
console.log(
`正在抓取第${CURRY_PAGENUMBER}页 ==>`,
requestQueue[index].title,
requestQueue[index].url
);
await loadHtml(requestQueue[index].url, requestQueue[index].title);
}
console.log(`第${CURRY_PAGENUMBER}页全部抓取完成`);
CURRY_PAGENUMBER++;
if (CURRY_PAGENUMBER <= MAX_PAGENUMBER) spiderQueue(soureUrl);
});
};
// 加载 HTML
const loadHtml = async (url, title) => {
try {
const { data } = await request({ url });
const $ = cheerio.load(data);
const soureURL = url.substring(0, 41);
const forNum = $(".paging a").length - 3;
const loadQueue = [];
for (let index = 1; index <= forNum; index++) {
const imageUrl = await loadImages(`${soureURL}-${index}.html`);
loadQueue.push(...imageUrl);
}
console.log(loadQueue);
await saveImages({ [title]: loadQueue }, loadQueue.length, "Cos");
} catch (error) {
console.log(`loadHtml: 下载${title}时出现错误！`);
console.log(error);
}
};
const loadImages = async (url) => {
try {
const resultHtml = await request({ url });
const $ = cheerio.load(resultHtml.data);
const imagesQueue = [];
$(".picture p img").each((i, elem) => {
imagesQueue.push($(elem).attr("src"));
});
return imagesQueue;
} catch (error) {
console.log(`loadImages: 下载${url}时出现错误！`);
console.log(error);
}
};
spiderQueue("https://www.xinwenba.net/plus/list-20020-1.html");
Write Preview
Loading…
Cancel
Save