ItailorCode

package resource;

import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.server.ServerProperties;

import javax.ws.rs.ApplicationPath;

/**
* Created by liker on 24/07/2015 0024.
* Group iTailor.hunters.neu.edu.cn
*/
@ApplicationPath("itailor/*")
public class ApplicationConfig extends ResourceConfig {
public ApplicationConfig() {
packages("resource");
register(JacksonJsonProvider.class);
property(ServerProperties.WADL_FEATURE_DISABLE,false);
}
}

package resource;

/**
* Created by liker on 08/08/2015 0008.
* Group iTailor.hunters.neu.edu.cn
*/
import org.apache.log4j.Logger;
import org.glassfish.jersey.message.internal.HeaderUtils;
import javax.ws.rs.client.ClientRequestContext;
import javax.ws.rs.client.ClientRequestFilter;
import javax.ws.rs.client.ClientResponseContext;
import javax.ws.rs.client.ClientResponseFilter;
import javax.ws.rs.container.*;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

@PreMatching
public class RequestLogFilter implements ContainerRequestFilter, ClientRequestFilter, ContainerResponseFilter, ClientResponseFilter {
private static final Logger LOGGER = Logger.getLogger(RequestLogFilter.class);
private static final String NOTIFICATION_PREFIX = "* ";
private static final String SERVER_REQUEST = "> ";
private static final String SERVER_RESPONSE = "< ";
private static final String CLIENT_REQUEST = "/ ";
private static final String CLIENT_RESPONSE = "\\ ";
private static final AtomicLong logSequence = new AtomicLong(0);

private StringBuilder prefixId(StringBuilder b, long id) {
b.append(Long.toString(id)).append(" ");
return b;
}

private void printRequestLine(final String prefix, StringBuilder b, long id, String method, URI uri) {
prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Request received on thread ").append(Thread.currentThread().getName()).append("\n");
prefixId(b, id).append(prefix).append(method).append(" ").append(uri.toASCIIString()).append("\n");
}

private void printResponseLine(final String prefix, StringBuilder b, long id, int status) {
prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Response received on thread ").append(Thread.currentThread().getName()).append("\n");
prefixId(b, id).append(prefix).append(Integer.toString(status)).append("\n");
}

private void printPrefixedHeaders(final String prefix, StringBuilder b, long id, MultivaluedMap<String, String> headers) {
for (Map.Entry<String, List<String>> e : headers.entrySet()) {
List<?> val = e.getValue();
String header = e.getKey();

if (val.size() == 1) {
prefixId(b, id).append(prefix).append(header).append(": ").append(val.get(0)).append("\n");
} else {
StringBuilder sb = new StringBuilder();
boolean add = false;
for (Object s : val) {
if (add) {
sb.append(',');
}
add = true;
sb.append(s);
}
prefixId(b, id).append(prefix).append(header).append(": ").append(sb.toString()).append("\n");
}
}
}

@Override
public void filter(ClientRequestContext context) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printRequestLine(CLIENT_REQUEST, b, id, context.getMethod(), context.getUri());
printPrefixedHeaders(CLIENT_REQUEST, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(context.getHeaders()));
LOGGER.debug(b.toString());
}

@Override
public void filter(ClientRequestContext requestContext, ClientResponseContext responseContext) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printResponseLine(CLIENT_RESPONSE, b, id, responseContext.getStatus());
printPrefixedHeaders(CLIENT_RESPONSE, b, id, responseContext.getHeaders());
LOGGER.debug(b.toString());
}

@Override
public void filter(ContainerRequestContext context) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printRequestLine(SERVER_REQUEST, b, id, context.getMethod(), context.getUriInfo().getRequestUri());
printPrefixedHeaders(SERVER_REQUEST, b, id, context.getHeaders());
LOGGER.debug(b.toString());
}

@Override
public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printResponseLine(SERVER_RESPONSE, b, id, responseContext.getStatus());
printPrefixedHeaders(SERVER_RESPONSE, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(responseContext.getHeaders()));
LOGGER.debug(b.toString());
}
}

<pre name="code" class="java">package resource;

/**
* Created by liker on 08/08/2015 0008.
* Group iTailor.hunters.neu.edu.cn
*/
import org.apache.log4j.Logger;
import org.glassfish.jersey.message.internal.HeaderUtils;
import javax.ws.rs.client.ClientRequestContext;
import javax.ws.rs.client.ClientRequestFilter;
import javax.ws.rs.client.ClientResponseContext;
import javax.ws.rs.client.ClientResponseFilter;
import javax.ws.rs.container.*;
import javax.ws.rs.core.MultivaluedMap;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

@PreMatching
public class RequestLogFilter implements ContainerRequestFilter, ClientRequestFilter, ContainerResponseFilter, ClientResponseFilter {
private static final Logger LOGGER = Logger.getLogger(RequestLogFilter.class);
private static final String NOTIFICATION_PREFIX = "* ";
private static final String SERVER_REQUEST = "> ";
private static final String SERVER_RESPONSE = "< ";
private static final String CLIENT_REQUEST = "/ ";
private static final String CLIENT_RESPONSE = "\\ ";
private static final AtomicLong logSequence = new AtomicLong(0);

private StringBuilder prefixId(StringBuilder b, long id) {
b.append(Long.toString(id)).append(" ");
return b;
}

private void printRequestLine(final String prefix, StringBuilder b, long id, String method, URI uri) {
prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Request received on thread ").append(Thread.currentThread().getName()).append("\n");
prefixId(b, id).append(prefix).append(method).append(" ").append(uri.toASCIIString()).append("\n");
}

private void printResponseLine(final String prefix, StringBuilder b, long id, int status) {
prefixId(b, id).append(NOTIFICATION_PREFIX).append("AirLog - Response received on thread ").append(Thread.currentThread().getName()).append("\n");
prefixId(b, id).append(prefix).append(Integer.toString(status)).append("\n");
}

private void printPrefixedHeaders(final String prefix, StringBuilder b, long id, MultivaluedMap<String, String> headers) {
for (Map.Entry<String, List<String>> e : headers.entrySet()) {
List<?> val = e.getValue();
String header = e.getKey();

if (val.size() == 1) {
prefixId(b, id).append(prefix).append(header).append(": ").append(val.get(0)).append("\n");
} else {
StringBuilder sb = new StringBuilder();
boolean add = false;
for (Object s : val) {
if (add) {
sb.append(',');
}
add = true;
sb.append(s);
}
prefixId(b, id).append(prefix).append(header).append(": ").append(sb.toString()).append("\n");
}
}
}

@Override
public void filter(ClientRequestContext context) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printRequestLine(CLIENT_REQUEST, b, id, context.getMethod(), context.getUri());
printPrefixedHeaders(CLIENT_REQUEST, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(context.getHeaders()));
LOGGER.debug(b.toString());
}

@Override
public void filter(ClientRequestContext requestContext, ClientResponseContext responseContext) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printResponseLine(CLIENT_RESPONSE, b, id, responseContext.getStatus());
printPrefixedHeaders(CLIENT_RESPONSE, b, id, responseContext.getHeaders());
LOGGER.debug(b.toString());
}

@Override
public void filter(ContainerRequestContext context) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printRequestLine(SERVER_REQUEST, b, id, context.getMethod(), context.getUriInfo().getRequestUri());
printPrefixedHeaders(SERVER_REQUEST, b, id, context.getHeaders());
LOGGER.debug(b.toString());
}

@Override
public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException {
long id = logSequence.incrementAndGet();
StringBuilder b = new StringBuilder();
printResponseLine(SERVER_RESPONSE, b, id, responseContext.getStatus());
printPrefixedHeaders(SERVER_RESPONSE, b, id, /*HeadersFactory*/HeaderUtils.asStringHeaders(responseContext.getHeaders()));
LOGGER.debug(b.toString());
}
}

package resource.community;

import hibernate.community.Account;
import resource.community.json.AccountJson;
import services.authority.AccessController;
import services.community.AccountService;

import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;

/**
* Created by liker on 16/08/2015 0016.
* Group iTailor.hunters.neu.edu.cn
*/
@Path("Account")
public class AccountResource {
//社区账号注册  OK OK
@POST
@Context
public boolean registerAccount(@QueryParam("email") final String email,
@HeaderParam("password") final String password) {
return AccountService.registerNewAccount(email, password);
}

//社区账号信息查询  OK
@GET
@Produces(MediaType.APPLICATION_JSON)
public AccountJson searchAccount(@QueryParam("accountID") final int accountID) {
Account account = AccountService.popOutAccount(accountID);
if (account != null) {
AccountJson accountJson = account.becomeToJson();
accountJson.setPassword("");
accountJson.setAuthenticate("");
return accountJson;
}
throw new WebApplicationException(400);
}

//修改社区账号
@PUT
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public boolean modifyAccount(@QueryParam("accountID") final int accountID,
@HeaderParam("authenticate") final String authenticate,
final AccountJson accountJson) {
if (accountJson == null) {
return false;
}
if (AccessController.isOperationValid(accountID, authenticate)) {
services.community.AccountService.popOutAccount(accountID).mergerFromJson(accountJson);
return true;
}
throw new WebApplicationException(400);
}

//注销社区账户（保留数据）
@DELETE
@Produces(MediaType.TEXT_PLAIN)
public boolean deleteAccount(@QueryParam("accountID") final int accountID,
@HeaderParam("authenticate") final String authenticate) {
if (AccessController.isOperationValid(accountID, authenticate)) {
AccountService.deleteAccount(accountID);
return true;
}
throw new WebApplicationException(400);
}
}

package crawler;

import hibernate.recommendation.ClothingImage;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static java.lang.Float.parseFloat;

/**
* Created by liker on 02/08/2015 0002.
* Group iTailor.hunters.neu.edu.cn
*/
public class ItemMaintainer implements PageProcessor {
private static Logger logger = Logger.getLogger(ItemMaintainer.class);
private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
private Item item;
private boolean downImage = true;

public ItemMaintainer(Item item) {
this.item = item;
}

@Override
public void process(Page page) {
Html html = page.getHtml();
try {
item.setItemName(html.xpath("//*[@id=\"J_DetailMeta\"]/div[1]/div[1]/div/div[1]/h1/text()").toString());
item.setKeyName(html.xpath("//*[@id=\"J_DetailMeta\"]/div[1]/div[1]/div/div[1]/p/text()").toString());
if (html.xpath("//*[@id=\"shopExtra\"]/div[1]/a/strong/text()").toString().isEmpty()) {
item.setShopName(html.xpath("//*[@id=\"side-shop-info\"]/div/h3/div/a/text()").toString());
}
item.setSaleQuantityInAMonth(getSellCount());
item.setProperties(getProperties(html));
//            item.setItemDescription(new ItemDescription(getDescriptionMap(html)));
getAllSkuItems(html);
getItemImagesFromTamll(page);
} catch (Exception e) {
e.printStackTrace();
}
}

@Override
public Site getSite() {
return site;
}

public void maintain(boolean downloadImage) {
downImage = downloadImage;
logger.debug("ITEM :" + item.getItemId() + " BEGIN to maintain.");
Spider.create(this).addUrl("https://detail.tmall.com/item.htm?id=" + item.getItemId()).thread(5).run();
logger.debug("ITEM :" + item.getItemId() + " END to maintain.");
}

Map<String, String> getDescriptionMap(Html html) {
int limit = html.xpath("///*[@id=\"J_AttrUL\"]/li").nodes().size();
String[] keys;
Map<String, String> result = new HashMap<>();
for (int i = 0; i < limit; i++) {
int j = i + 1;
String one = html.xpath("///*[@id=\"J_AttrUL\"]/li[" + j + "]/text()").toString().replaceAll("\u00A0", "");
String[] key = one.split(":", 2);
result.put(key[0], key[1]);
}
return result;
}

List<Property> getProperties(Html html) {
int limit = html.xpath("///*[@id=\"J_AttrUL\"]/li").nodes().size();
List<Property> properties = new ArrayList<>();
String[] keys;
for (int i = 0; i < limit; i++) {
int j = i + 1;
String one = html.xpath("///*[@id=\"J_AttrUL\"]/li[" + j + "]/text()").toString().replaceAll("\u00A0", "");
String[] key = one.split(":", 2);
properties.add(new Property(key[0], key[1]));
}
return properties;
}

void getAllSkuItems(Html html) throws Exception {
Matcher m = Pattern.compile("\\{\"api\":(.*)\\}")
.matcher(html.css("#J_DetailMeta > div.tm-clear > script").toString());
if (m.find()) {
try {
JSONObject one = new JSONObject(m.group()).getJSONObject("valItemInfo");
JSONArray two = one.getJSONArray("skuList");
JSONObject three = one.getJSONObject("skuMap");

for (int i = 0; i < two.length(); i++) {
String sku = String.valueOf(two.getJSONObject(i).getLong("skuId"));
String[] temp = two.getJSONObject(i).getString("names").split(" ", 2);
String size = temp[0];
String color = temp[1].replace(" ", "");
String pvs = two.getJSONObject(i).getString("pvs");
float price = getTmallRealPrice(sku);
int stock = three.getJSONObject(";" + pvs + ";").getInt("stock");
int sell = getSellCount();
item.getSkuItems().add(new SkuItem(sku, color, size, stock, sell, price));
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

public void setSite(Site site) {
this.site = site;
}

public Item getItem() {
return item;
}

public void setItem(Item item) {
this.item = item;
}

public float getTmallRealPrice(String skuId) throws IOException, JSONException {
String itemId = item.getItemId();
HttpClientBuilder builder = HttpClients.custom();
builder.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36");
CloseableHttpClient httpClient = builder.build();
HttpGet httpGet = new HttpGet("https://mdskip.taobao.com/core/initItemDetail.htm?itemId=" + itemId);
httpGet.addHeader("scheme", "https");
httpGet.addHeader("version", "HTTP/1.1");
httpGet.addHeader("accept", "*/*");
httpGet.addHeader("accept-encoding", "gzip, deflate, sdch");
httpGet.addHeader("accept-language", "en,zh-CN;q=0.8,zh;q=0.6");
httpGet.addHeader("cache-control", "no-cache");
httpGet.addHeader("Referer", "https://detail.tmall.com/item.htm?id=" + itemId);
CloseableHttpResponse response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
OutputStream out = new ByteArrayOutputStream();
if (entity != null) {
InputStream instream = entity.getContent();
int len;
byte[] tmp = new byte[2048];
while ((len = instream.read(tmp)) != -1) {
out.write(tmp, 0, len);
}
instream.close();
} else {
out.write(new byte[]{'{', '}'});
}
out.close();
String context = out.toString();
String realPrice = new JSONObject(context)
.getJSONObject("defaultModel")
.getJSONObject("itemPriceResultDO")
.getJSONObject("priceInfo")
.getJSONObject(skuId)
.getJSONArray("promotionList")
.getJSONObject(0)
.getString("price");
response.close();
httpClient.close();
return parseFloat(realPrice);
}

public int getSellCount() throws IOException, JSONException {
String itemId = item.getItemId();
HttpClientBuilder builder = HttpClients.custom();
builder.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36");
CloseableHttpClient httpClient = builder.build();
HttpGet httpGet = new HttpGet("https://mdskip.taobao.com/core/initItemDetail.htm?itemId=" + itemId);
httpGet.addHeader("scheme", "https");
httpGet.addHeader("version", "HTTP/1.1");
httpGet.addHeader("accept", "*/*");
httpGet.addHeader("accept-encoding", "gzip, deflate, sdch");
httpGet.addHeader("accept-language", "en,zh-CN;q=0.8,zh;q=0.6");
httpGet.addHeader("cache-control", "no-cache");
httpGet.addHeader("Referer", "https://detail.tmall.com/item.htm?id=" + itemId);
CloseableHttpResponse response = httpClient.execute(httpGet);
HttpEntity entity = response.getEntity();
OutputStream out = new ByteArrayOutputStream();
if (entity != null) {
InputStream instream = entity.getContent();
int len;
byte[] tmp = new byte[2048];
while ((len = instream.read(tmp)) != -1) {
out.write(tmp, 0, len);
}
instream.close();
} else {
out.write(new byte[]{'{', '}'});
}
int sellCount = new JSONObject(out.toString())
.getJSONObject("defaultModel")
.getJSONObject("sellCountDO")
.getInt("sellCount");
out.close();
response.close();
httpClient.close();
return sellCount;
}

public void getItemImagesFromTamll(Page page) {
int limit = page.getHtml().xpath("//*[@id=\"J_UlThumb\"]/li").nodes().size();
ClothingImage[] clothingImages;
for (int i = 0; i < limit; i++) {
int j = i + 1;
clothingImages = new ClothingImage[limit];
clothingImages[i] = new ClothingImage();
clothingImages[i].setSource(page.getHtml()
.xpath("//*[@id=\"J_UlThumb\"]/li[" + j + "]/a/img/@src")
.toString()
.replace("//", "")
.replace("_60x60q90.jpg", ""));
item.getClothingImages().add(clothingImages[i]);
if(downImage){
new GetImageFromWeb().getImage(item.getItemId() + "@" + i, clothingImages[i]);
}
}
}

}

package services.daos;

import org.hibernate.Session;
import util.hibernate.HibernateSessionFactory;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

public class BaseDAO<T> {
public void create(T object) {

Session session =  HibernateSessionFactory.getSessionFactory().openSession();
try {
session.beginTransaction();
session.persist(object);
session.getTransaction().commit();
session.flush();
} catch (Exception e) {
session.getTransaction().rollback();
} finally {
session.close();
}
}

public void update(T object) {
Session session = HibernateSessionFactory.getSessionFactory().openSession();
try {
session.beginTransaction();
session.update(object);
session.getTransaction().commit();
session.flush();
} catch (Exception e) {
session.getTransaction().rollback();
} finally {
session.close();
}
}

public void delete(T object) {
Session session = HibernateSessionFactory.getSessionFactory().openSession();
try {
session.beginTransaction();
session.delete(object);
session.getTransaction().commit();
session.flush();
} catch (Exception e) {
session.getTransaction().rollback();
} finally {
session.close();
}
}

public List<T> list(String hql) {
Session session = HibernateSessionFactory.getSessionFactory().openSession();
try {
session.beginTransaction();
return session.createQuery(hql).list();
} finally {
session.getTransaction().commit();
session.close();
}
}

public static List<String> takeOutAStringListFromDB(String property, String entity) {
List<String> properties = new ArrayList<>();
Session session = HibernateSessionFactory.getSessionFactory().openSession();
try {
String hql = "select c." + property + " from " + entity + " c";
properties = session.createQuery(hql).list();
Iterator<String> iterator = properties.iterator();
while (iterator.hasNext()) {
if (iterator.next().equals("")) {
iterator.remove();
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
session.close();
return properties;
}
}
public static List<Integer> takeOutAIntegerListFromDB(String property, String entity) {
List<Integer> properties = new ArrayList<>();
Session session = HibernateSessionFactory.getSessionFactory().openSession();
try {
String hql = "select c." + property + " from " + entity + " c";
properties = session.createQuery(hql).list();
} catch (Exception e) {
e.printStackTrace();
} finally {
session.close();
return properties;
}
}

}