我正试图更快地调用多个API。
在下面的代码中，// Import required modules
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const url = "http://johndevisser.marktplaza.nl/?p=";
async function getProductsHtml(data) {
const $ = await cheerio.load(data);
let productsHTML = [];
$("div.item").each((i,prod) => {
productsHTML.push(($(prod).html()));
});
return productsHTML;
};
async function parseProducts(html) {
let products = [];
for (item in html) {
// Store the data we already have
const $ = await cheerio.load(html[item]);
let product = {};
let mpUrl = $("a").attr("href");
product["title"] = $("a").attr("title");
product["mpUrl"] = mpUrl;
product["imgUrl"] = $("img").attr("src");
let priceText = $("span.subtext").text().split("\xa0")[1].replace(",",".");
product["price"] = parseFloat(priceText);
products.push(product);
}
return products;
}
async function addDescriptionToProducts(prods) {
for (i in prods) {
const response = await axios.get(prods[i]["mpUrl"])
const $ = cheerio.load(response.data);
description = $("div.description p").text();
prods[i]["descr"] = description;
}
return prods
}
async function getProductsFromPage(i) {
try {
const page = await axios.get(`http://johndevisser.marktplaza.nl/?p=${i}`);
console.log("GET request succeeded!");
// Get the Array with HTML of each product
const productsHTML = await getProductsHtml(page.data);
console.log("HTML array obtained!");
// Get the Array of objects with meta info
const productsParsed = await parseProducts(productsHTML);
console.log("Products parsed!")
// Add description to each product
const productsMeta = await addDescriptionToProducts(productsParsed);
console.log("Descriptions added!")
// Return the Array with all product information
return productsMeta;
} catch(e) {
console.log(e);
}
};
async function saveAllProducts() {
try {
const allProducts = await getallProducts();
let jsonProducts = await JSON.stringify(allProducts);
fs.writeFile("products.json",jsonProducts,"utf8",(e) => {
if (e) {
console.log(e)
}
});
} catch(e) {
console.log(e);
}
}
async function getallProducts() {
try {
let allProducts = [];
for (let i = 1; i < 855; i++) {
const productsFromPage = await getProductsFromPage(i);
allProducts = [...allProducts,...productsFromPage];
console.log("Saved products from page " + i);
}
return allProducts
} catch(e) {
console.log(e);
}
}
saveAllProducts();
是当前同步的版本。我有一种感觉，getFilteredEvents操作将等待每个API的响应（内部使用map(x -> x.getFilteredEvents(eventResearch))，然后传递给下一个API，以构建要返回的RestTemplate.exchange()。一种解决方案可能是在单独的线程上启动List<Event>调用，但是我想尝试map API。
因此CompletableFuture是我努力改善响应时间的结果。
getFilteredEventsFaster我的理解是，我希望将@Service
public class EventsresearchService {
@Autowired
private List<UniformEventsresearchApi> eventsresearchApis;
// this works,but I'm trying to improve it
public EventResearchResponse getFilteredEvents(EventResearch eventResearch) {
List<Event> eventsList = eventsresearchApis
.stream()
.map(x -> x.getFilteredEvents(eventResearch))
.flatMap(List::stream)
.collect(Collectors.toList());
return extractResponse(eventResearch,eventsList);
}
// this doesn't work yet: what is wrong?
public CompletableFuture<List<Event>> getFilteredEventsFaster(EventResearch eventResearch) {
List<CompletableFuture<List<Event>>> futureEventsList = eventsresearchApis
.parallelStream()
.map(x -> CompletableFuture.supplyAsync(() -> x.getFilteredEvents(eventResearch)))
.collect(Collectors.toList());
return CompletableFuture.allOf(futureEventsList.toArray(new CompletableFuture<List<Event>>[0]));
}
}
而不是CompletableFuture<List<Event>>发送回我的前端，因此，List<CompletableFuture<List<Event>>>调用（如果我理解正确，类似于CompletableFuture.allOf()操作，从多个flatmap创建一个CompletableFuture。
不幸的是，使用CompleteableFuture时出现Generic array creation编译错误。
我在做什么错了？
我觉得使用new CompletableFuture<List<Event>>[0]方法确实可以让我收集所有答案，但是那将是对Service线程的阻塞操作，不是吗？ （如果我理解正确的话，这样做会挫败试图将join返回到我的前端的目的。）