词法分析代码

import java.util.List;

import test.Analysis;

public class Main {

 /**
  * @param args
  */
 public static void main(String[] args) {
  // TODO Auto-generated method stub
  
  Analysis an = new Analysis();
  
  List list = an.scan("http://www.google.cn/search?hl=zh-CN&newwindow=1&q=fy+%E5%88%A4%E6%96%AD&aq=f&oq=&aqi=");
  
  an.print(list);

 }

}

 

 

 

 

package test;

public class Item {
 private int type = 0;
 private int length = 0;
 private String content = "";
 
 public int getType() {
  return type;
 }
 public void setType(int type) {
  this.type = type;
 }
 public int getLength() {
  return length;
 }
 public void setLength(int length) {
  this.length = length;
 }
 public String getContent() {
  return content;
 }
 public void setContent(String content) {
  this.content = content;
 }
 
 public String toString() {
  StringBuilder builder = new StringBuilder();
  
  builder.append("type : ").append(type).append("/n");
  builder.append("length : ").append(length).append("/n");
  builder.append("content : ").append(content).append("/n");
  
  return builder.toString();
 }
}

 

 

 

package test;

import java.util.ArrayList;
import java.util.List;

public class Analysis {

 public List<Item> scan(String content) {
  List<Item> result = new ArrayList<Item>();
  StringBuilder builder = new StringBuilder();
  int old = 0;
  int temp = 0;
  char c;
  
  int s = 0;
  
  s = content.length();
  
  for(int i=0;i<s;i++) {
   c = content.charAt(i);
   temp = this.decide(c);
   if(old == 0) {
    old = temp;
    builder.append(c);
   } else {
    if(old == temp) {
     builder.append(c);
    } else {
     if( old == temp || (old == 2 && temp == 1) || (old == 2 && temp == 3)) {
      builder.append(c);
     } else {
      Item item = new Item();
      item.setType(old);
      item.setContent(builder.toString());
      item.setLength(item.getContent().length());
      result.add(item);
      //result.add(builder.toString());
      builder.delete(0, builder.length());
      builder.append(c);
      old = temp;
     }
    }
   }
  }

  if(builder.length() > 0) {
   Item item = new Item();
   item.setType(old);
   item.setContent(builder.toString());
   item.setLength(item.getContent().length());
   result.add(item);
   //result.add(builder.toString());
  }
  return result;
 }
 
 public int decide(char c) {
  int result = 0;
  
  if(c >= '0' && c <= '9') {
   result = 1;
  } else if(c >= 'A' && c <= 'Z') {
   result = 2;
  } else if(c >= 'a' && c <= 'z') {
   result = 2;
  } else if(c > 0 && c < 127){
   result = c;
  } else {
   result = 3;
  }
  
  return result;
 }
 
 
 public void print(List<Item> result) {
  int s = result.size();
  for(int i=0;i<s;i++) {
   System.out.println(result.get(i));
  }
 }
}