ItailorCode
来源：互联网 发布：淘宝店铺全屏装修教程 编辑：程序博客网 时间：2024/05/05 05:42
package resource;import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider;import org.glassfish.jersey.server.ResourceConfig;import org.glassfish.jersey.server.ServerProperties;import javax.ws.rs.ApplicationPath;/** * Created by liker on 24/07/2015 0024. * Group iTailor.hunters.neu.edu.cn */@ApplicationPath("itailor/*")public class ApplicationConfig extends ResourceConfig { public ApplicationConfig() { packages("resource"); register(JacksonJsonProvider.class); property(ServerProperties.WADL_FEATURE_DISABLE,false); }}package resource;/** * Created by liker on 08/08/2015 0008. * Group iTailor.hunters.neu.edu.cn */import org.apache.log4j.Logger;import org.glassfish.jersey.message.internal.HeaderUtils;import javax.ws.rs.client.ClientRequestContext;import javax.ws.rs.client.ClientRequestFilter;import javax.ws.rs.client.ClientResponseContext;import javax.ws.rs.client.ClientResponseFilter;import javax.ws.rs.container.*;import javax.ws.rs.core.MultivaluedMap;import java.io.IOException;import java.net.URI;import java.util.List;import java.util.Map;import java.util.concurrent.atomic.AtomicLong;@PreMatchingpublic class RequestLogFilter implements ContainerRequestFilter, ClientRequestFilter, ContainerResponseFilter, ClientResponseFilter { private static final Logger LOGGER = Logger.getLogger(RequestLogFilter.class); private static final String NOTIFICATION_PREFIX = "* "; private static final String SERVER_REQUEST = "> "; private static final String SERVER_RESPONSE = "< "; private static final String CLIENT_REQUEST = "/ "; private static final String CLIENT_RESPONSE = "\\ "; private static final AtomicLong logSequence = new AtomicLong(0); private StringBuilder prefixId(StringBuilder b, long id) { b.append(Long.toString(id)).append(" "); return b; } private void printRequestLine(final String prefix, StringBuilder b, long id, String method, URI uri) { prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Request received on thread ").append(Thread.currentThread().getName()).append("\n"); prefixId(b, id).append(prefix).append(method).append(" ").append(uri.toASCIIString()).append("\n"); } private void printResponseLine(final String prefix, StringBuilder b, long id, int status) { prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Response received on thread ").append(Thread.currentThread().getName()).append("\n"); prefixId(b, id).append(prefix).append(Integer.toString(status)).append("\n"); } private void printPrefixedHeaders(final String prefix, StringBuilder b, long id, MultivaluedMap<String, String> headers) { for (Map.Entry<String, List<String>> e : headers.entrySet()) { List<?> val = e.getValue(); String header = e.getKey(); if (val.size() == 1) { prefixId(b, id).append(prefix).append(header).append(": ").append(val.get(0)).append("\n"); } else { StringBuilder sb = new StringBuilder(); boolean add = false; for (Object s : val) { if (add) { sb.append(','); } add = true; sb.append(s); } prefixId(b, id).append(prefix).append(header).append(": ").append(sb.toString()).append("\n"); } } } @Override public void filter(ClientRequestContext context) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printRequestLine(CLIENT_REQUEST, b, id, context.getMethod(), context.getUri()); printPrefixedHeaders(CLIENT_REQUEST, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(context.getHeaders())); LOGGER.debug(b.toString()); } @Override public void filter(ClientRequestContext requestContext, ClientResponseContext responseContext) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printResponseLine(CLIENT_RESPONSE, b, id, responseContext.getStatus()); printPrefixedHeaders(CLIENT_RESPONSE, b, id, responseContext.getHeaders()); LOGGER.debug(b.toString()); } @Override public void filter(ContainerRequestContext context) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printRequestLine(SERVER_REQUEST, b, id, context.getMethod(), context.getUriInfo().getRequestUri()); printPrefixedHeaders(SERVER_REQUEST, b, id, context.getHeaders()); LOGGER.debug(b.toString()); } @Override public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printResponseLine(SERVER_RESPONSE, b, id, responseContext.getStatus()); printPrefixedHeaders(SERVER_RESPONSE, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(responseContext.getHeaders())); LOGGER.debug(b.toString()); }}<pre name="code" class="java">package resource;/** * Created by liker on 08/08/2015 0008. * Group iTailor.hunters.neu.edu.cn */import org.apache.log4j.Logger;import org.glassfish.jersey.message.internal.HeaderUtils;import javax.ws.rs.client.ClientRequestContext;import javax.ws.rs.client.ClientRequestFilter;import javax.ws.rs.client.ClientResponseContext;import javax.ws.rs.client.ClientResponseFilter;import javax.ws.rs.container.*;import javax.ws.rs.core.MultivaluedMap;import java.io.IOException;import java.net.URI;import java.util.List;import java.util.Map;import java.util.concurrent.atomic.AtomicLong;@PreMatchingpublic class RequestLogFilter implements ContainerRequestFilter, ClientRequestFilter, ContainerResponseFilter, ClientResponseFilter { private static final Logger LOGGER = Logger.getLogger(RequestLogFilter.class); private static final String NOTIFICATION_PREFIX = "* "; private static final String SERVER_REQUEST = "> "; private static final String SERVER_RESPONSE = "< "; private static final String CLIENT_REQUEST = "/ "; private static final String CLIENT_RESPONSE = "\\ "; private static final AtomicLong logSequence = new AtomicLong(0); private StringBuilder prefixId(StringBuilder b, long id) { b.append(Long.toString(id)).append(" "); return b; } private void printRequestLine(final String prefix, StringBuilder b, long id, String method, URI uri) { prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Request received on thread ").append(Thread.currentThread().getName()).append("\n"); prefixId(b, id).append(prefix).append(method).append(" ").append(uri.toASCIIString()).append("\n"); } private void printResponseLine(final String prefix, StringBuilder b, long id, int status) { prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Response received on thread ").append(Thread.currentThread().getName()).append("\n"); prefixId(b, id).append(prefix).append(Integer.toString(status)).append("\n"); } private void printPrefixedHeaders(final String prefix, StringBuilder b, long id, MultivaluedMap<String, String> headers) { for (Map.Entry<String, List<String>> e : headers.entrySet()) { List<?> val = e.getValue(); String header = e.getKey(); if (val.size() == 1) { prefixId(b, id).append(prefix).append(header).append(": ").append(val.get(0)).append("\n"); } else { StringBuilder sb = new StringBuilder(); boolean add = false; for (Object s : val) { if (add) { sb.append(','); } add = true; sb.append(s); } prefixId(b, id).append(prefix).append(header).append(": ").append(sb.toString()).append("\n"); } } } @Override public void filter(ClientRequestContext context) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printRequestLine(CLIENT_REQUEST, b, id, context.getMethod(), context.getUri()); printPrefixedHeaders(CLIENT_REQUEST, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(context.getHeaders())); LOGGER.debug(b.toString()); } @Override public void filter(ClientRequestContext requestContext, ClientResponseContext responseContext) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printResponseLine(CLIENT_RESPONSE, b, id, responseContext.getStatus()); printPrefixedHeaders(CLIENT_RESPONSE, b, id, responseContext.getHeaders()); LOGGER.debug(b.toString()); } @Override public void filter(ContainerRequestContext context) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printRequestLine(SERVER_REQUEST, b, id, context.getMethod(), context.getUriInfo().getRequestUri()); printPrefixedHeaders(SERVER_REQUEST, b, id, context.getHeaders()); LOGGER.debug(b.toString()); } @Override public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException { long id = logSequence.incrementAndGet(); StringBuilder b = new StringBuilder(); printResponseLine(SERVER_RESPONSE, b, id, responseContext.getStatus()); printPrefixedHeaders(SERVER_RESPONSE, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(responseContext.getHeaders())); LOGGER.debug(b.toString()); }}package resource.community;import hibernate.community.Account;import resource.community.json.AccountJson;import services.authority.AccessController;import services.community.AccountService;import javax.ws.rs.*;import javax.ws.rs.core.Context;import javax.ws.rs.core.MediaType;/** * Created by liker on 16/08/2015 0016. * Group iTailor.hunters.neu.edu.cn */@Path("Account")public class AccountResource { //社区账号注册 OK OK @POST @Context public boolean registerAccount(@QueryParam("email") final String email, @HeaderParam("password") final String password) { return AccountService.registerNewAccount(email, password); } //社区账号信息查询 OK @GET @Produces(MediaType.APPLICATION_JSON) public AccountJson searchAccount(@QueryParam("accountID") final int accountID) { Account account = AccountService.popOutAccount(accountID); if (account != null) { AccountJson accountJson = account.becomeToJson(); accountJson.setPassword(""); accountJson.setAuthenticate(""); return accountJson; } throw new WebApplicationException(400); } //修改社区账号 @PUT @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public boolean modifyAccount(@QueryParam("accountID") final int accountID, @HeaderParam("authenticate") final String authenticate, final AccountJson accountJson) { if (accountJson == null) { return false; } if (AccessController.isOperationValid(accountID, authenticate)) { services.community.AccountService.popOutAccount(accountID).mergerFromJson(accountJson); return true; } throw new WebApplicationException(400); } //注销社区账户（保留数据） @DELETE @Produces(MediaType.TEXT_PLAIN) public boolean deleteAccount(@QueryParam("accountID") final int accountID, @HeaderParam("authenticate") final String authenticate) { if (AccessController.isOperationValid(accountID, authenticate)) { AccountService.deleteAccount(accountID); return true; } throw new WebApplicationException(400); }}package crawler;import hibernate.recommendation.ClothingImage;import org.apache.http.HttpEntity;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClientBuilder;import org.apache.http.impl.client.HttpClients;import org.apache.log4j.Logger;import org.json.JSONArray;import org.json.JSONException;import org.json.JSONObject;import us.codecraft.webmagic.Page;import us.codecraft.webmagic.Site;import us.codecraft.webmagic.Spider;import us.codecraft.webmagic.processor.PageProcessor;import us.codecraft.webmagic.selector.Html;import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.regex.Matcher;import java.util.regex.Pattern;import static java.lang.Float.parseFloat;/** * Created by liker on 02/08/2015 0002. * Group iTailor.hunters.neu.edu.cn */public class ItemMaintainer implements PageProcessor { private static Logger logger = Logger.getLogger(ItemMaintainer.class); private Site site = Site.me().setRetryTimes(3).setSleepTime(1000); private Item item; private boolean downImage = true; public ItemMaintainer(Item item) { this.item = item; } @Override public void process(Page page) { Html html = page.getHtml(); try { item.setItemName(html.xpath("//*[@id=\"J_DetailMeta\"]/div[1]/div[1]/div/div[1]/h1/text()").toString()); item.setKeyName(html.xpath("//*[@id=\"J_DetailMeta\"]/div[1]/div[1]/div/div[1]/p/text()").toString()); if (html.xpath("//*[@id=\"shopExtra\"]/div[1]/a/strong/text()").toString().isEmpty()) { item.setShopName(html.xpath("//*[@id=\"side-shop-info\"]/div/h3/div/a/text()").toString()); } item.setSaleQuantityInAMonth(getSellCount()); item.setProperties(getProperties(html));// item.setItemDescription(new ItemDescription(getDescriptionMap(html))); getAllSkuItems(html); getItemImagesFromTamll(page); } catch (Exception e) { e.printStackTrace(); } } @Override public Site getSite() { return site; } public void maintain(boolean downloadImage) { downImage = downloadImage; logger.debug("ITEM :" + item.getItemId() + " BEGIN to maintain."); Spider.create(this).addUrl("https://detail.tmall.com/item.htm?id=" + item.getItemId()).thread(5).run(); logger.debug("ITEM :" + item.getItemId() + " END to maintain."); } Map<String, String> getDescriptionMap(Html html) { int limit = html.xpath("///*[@id=\"J_AttrUL\"]/li").nodes().size(); String[] keys; Map<String, String> result = new HashMap<>(); for (int i = 0; i < limit; i++) { int j = i + 1; String one = html.xpath("///*[@id=\"J_AttrUL\"]/li[" + j + "]/text()").toString().replaceAll("\u00A0", ""); String[] key = one.split(":", 2); result.put(key[0], key[1]); } return result; } List<Property> getProperties(Html html) { int limit = html.xpath("///*[@id=\"J_AttrUL\"]/li").nodes().size(); List<Property> properties = new ArrayList<>(); String[] keys; for (int i = 0; i < limit; i++) { int j = i + 1; String one = html.xpath("///*[@id=\"J_AttrUL\"]/li[" + j + "]/text()").toString().replaceAll("\u00A0", ""); String[] key = one.split(":", 2); properties.add(new Property(key[0], key[1])); } return properties; } void getAllSkuItems(Html html) throws Exception { Matcher m = Pattern.compile("\\{\"api\":(.*)\\}") .matcher(html.css("#J_DetailMeta > div.tm-clear > script").toString()); if (m.find()) { try { JSONObject one = new JSONObject(m.group()).getJSONObject("valItemInfo"); JSONArray two = one.getJSONArray("skuList"); JSONObject three = one.getJSONObject("skuMap"); for (int i = 0; i < two.length(); i++) { String sku = String.valueOf(two.getJSONObject(i).getLong("skuId")); String[] temp = two.getJSONObject(i).getString("names").split(" ", 2); String size = temp[0]; String color = temp[1].replace(" ", ""); String pvs = two.getJSONObject(i).getString("pvs"); float price = getTmallRealPrice(sku); int stock = three.getJSONObject(";" + pvs + ";").getInt("stock"); int sell = getSellCount(); item.getSkuItems().add(new SkuItem(sku, color, size, stock, sell, price)); } } catch (Exception e) { e.printStackTrace(); } } } public void setSite(Site site) { this.site = site; } public Item getItem() { return item; } public void setItem(Item item) { this.item = item; } public float getTmallRealPrice(String skuId) throws IOException, JSONException { String itemId = item.getItemId(); HttpClientBuilder builder = HttpClients.custom(); builder.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36"); CloseableHttpClient httpClient = builder.build(); HttpGet httpGet = new HttpGet("https://mdskip.taobao.com/core/initItemDetail.htm?itemId=" + itemId); httpGet.addHeader("scheme", "https"); httpGet.addHeader("version", "HTTP/1.1"); httpGet.addHeader("accept", "*/*"); httpGet.addHeader("accept-encoding", "gzip, deflate, sdch"); httpGet.addHeader("accept-language", "en,zh-CN;q=0.8,zh;q=0.6"); httpGet.addHeader("cache-control", "no-cache"); httpGet.addHeader("Referer", "https://detail.tmall.com/item.htm?id=" + itemId); CloseableHttpResponse response = httpClient.execute(httpGet); HttpEntity entity = response.getEntity(); OutputStream out = new ByteArrayOutputStream(); if (entity != null) { InputStream instream = entity.getContent(); int len; byte[] tmp = new byte[2048]; while ((len = instream.read(tmp)) != -1) { out.write(tmp, 0, len); } instream.close(); } else { out.write(new byte[]{'{', '}'}); } out.close(); String context = out.toString(); String realPrice = new JSONObject(context) .getJSONObject("defaultModel") .getJSONObject("itemPriceResultDO") .getJSONObject("priceInfo") .getJSONObject(skuId) .getJSONArray("promotionList") .getJSONObject(0) .getString("price"); response.close(); httpClient.close(); return parseFloat(realPrice); } public int getSellCount() throws IOException, JSONException { String itemId = item.getItemId(); HttpClientBuilder builder = HttpClients.custom(); builder.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36"); CloseableHttpClient httpClient = builder.build(); HttpGet httpGet = new HttpGet("https://mdskip.taobao.com/core/initItemDetail.htm?itemId=" + itemId); httpGet.addHeader("scheme", "https"); httpGet.addHeader("version", "HTTP/1.1"); httpGet.addHeader("accept", "*/*"); httpGet.addHeader("accept-encoding", "gzip, deflate, sdch"); httpGet.addHeader("accept-language", "en,zh-CN;q=0.8,zh;q=0.6"); httpGet.addHeader("cache-control", "no-cache"); httpGet.addHeader("Referer", "https://detail.tmall.com/item.htm?id=" + itemId); CloseableHttpResponse response = httpClient.execute(httpGet); HttpEntity entity = response.getEntity(); OutputStream out = new ByteArrayOutputStream(); if (entity != null) { InputStream instream = entity.getContent(); int len; byte[] tmp = new byte[2048]; while ((len = instream.read(tmp)) != -1) { out.write(tmp, 0, len); } instream.close(); } else { out.write(new byte[]{'{', '}'}); } int sellCount = new JSONObject(out.toString()) .getJSONObject("defaultModel") .getJSONObject("sellCountDO") .getInt("sellCount"); out.close(); response.close(); httpClient.close(); return sellCount; } public void getItemImagesFromTamll(Page page) { int limit = page.getHtml().xpath("//*[@id=\"J_UlThumb\"]/li").nodes().size(); ClothingImage[] clothingImages; for (int i = 0; i < limit; i++) { int j = i + 1; clothingImages = new ClothingImage[limit]; clothingImages[i] = new ClothingImage(); clothingImages[i].setSource(page.getHtml() .xpath("//*[@id=\"J_UlThumb\"]/li[" + j + "]/a/img/@src") .toString() .replace("//", "") .replace("_60x60q90.jpg", "")); item.getClothingImages().add(clothingImages[i]); if(downImage){ new GetImageFromWeb().getImage(item.getItemId() + "@" + i, clothingImages[i]); } } }}package services.daos;import org.hibernate.Session;import util.hibernate.HibernateSessionFactory;import java.util.ArrayList;import java.util.Iterator;import java.util.List;public class BaseDAO<T> { public void create(T object) { Session session = HibernateSessionFactory.getSessionFactory().openSession(); try { session.beginTransaction(); session.persist(object); session.getTransaction().commit(); session.flush(); } catch (Exception e) { session.getTransaction().rollback(); } finally { session.close(); } } public void update(T object) { Session session = HibernateSessionFactory.getSessionFactory().openSession(); try { session.beginTransaction(); session.update(object); session.getTransaction().commit(); session.flush(); } catch (Exception e) { session.getTransaction().rollback(); } finally { session.close(); } } public void delete(T object) { Session session = HibernateSessionFactory.getSessionFactory().openSession(); try { session.beginTransaction(); session.delete(object); session.getTransaction().commit(); session.flush(); } catch (Exception e) { session.getTransaction().rollback(); } finally { session.close(); } } public List<T> list(String hql) { Session session = HibernateSessionFactory.getSessionFactory().openSession(); try { session.beginTransaction(); return session.createQuery(hql).list(); } finally { session.getTransaction().commit(); session.close(); } } public static List<String> takeOutAStringListFromDB(String property, String entity) { List<String> properties = new ArrayList<>(); Session session = HibernateSessionFactory.getSessionFactory().openSession(); try { String hql = "select c." + property + " from " + entity + " c"; properties = session.createQuery(hql).list(); Iterator<String> iterator = properties.iterator(); while (iterator.hasNext()) { if (iterator.next().equals("")) { iterator.remove(); } } } catch (Exception e) { e.printStackTrace(); } finally { session.close(); return properties; } } public static List<Integer> takeOutAIntegerListFromDB(String property, String entity) { List<Integer> properties = new ArrayList<>(); Session session = HibernateSessionFactory.getSessionFactory().openSession(); try { String hql = "select c." + property + " from " + entity + " c"; properties = session.createQuery(hql).list(); } catch (Exception e) { e.printStackTrace(); } finally { session.close(); return properties; } }} 0 0
