android之访问网络获取网页数据并存入数据库

工具：android studio

APK:http://fir.im/ksal

使用URLConnection 连接网络获取网页数据，把联网放在异步线程里执行 使用AsyncTask   

extends AsyncTask<String, Integer, StringBuffer> 在doInBackground 方法里执行联网的方法，
onPostExecute(StringBuffer result)  方法用于在执行完后台任务后更新UI,显示结果

连接网络有get 和 post 方法，

 public void doGetURL(String url) throws Exception {        URL localURL = new URL(url);        URLConnection connection = localURL.openConnection();        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;        httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");        if (httpURLConnection.getResponseCode() >= 300) {            throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());        }        inputStream = httpURLConnection.getInputStream();        inputStreamReader = new InputStreamReader(inputStream);        reader = new BufferedReader(inputStreamReader);        while ((tempLine = reader.readLine()) != null) {            tempHTML.append(tempLine);        }    }

    private void doPostURL(String url) throws Exception{        URL localURL = new URL(url);        URLConnection connection = localURL.openConnection();        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;        httpURLConnection.setDoOutput(true);        httpURLConnection.setRequestMethod("POST");        httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");        httpURLConnection.setRequestProperty("Content-Length", String.valueOf(url.length()));        outputStream = httpURLConnection.getOutputStream();        outputStreamWriter = new OutputStreamWriter(outputStream);        outputStreamWriter.write(url.toString());        outputStreamWriter.flush();        if (httpURLConnection.getResponseCode() >= 300) {            throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());        }        inputStream = httpURLConnection.getInputStream();        inputStreamReader = new InputStreamReader(inputStream);        reader = new BufferedReader(inputStreamReader);        while ((tempLine = reader.readLine()) != null) {            tempHTML.append(tempLine);        }    }

很多时候我们看的URL 地址都是有重定向的，这个好解决，自己递归或者使用URLConnection提供的方法，解决重定向问题，获取真实的地址

 /**     * 直接将重定向交给HttpURLConnection完成，调用它的setInstanceFollowRedirects(true)方法，     * 这样一来重定向对于外部来说是透明的，我们完全感知不到重定向的发生，     * 但是我们没有办法截获重定向的过程，并且对于重定向次数有限制，如果超过4次的重定向，后续的重定向将会被忽略。     * */    public String redirectGetPath(final String str)            throws MalformedURLException {        URL url = new URL(str);        String realURL = null;        HttpURLConnection conn =  null;        try {            conn = (HttpURLConnection) url.openConnection();            conn.setConnectTimeout(30000);            conn.setReadTimeout(30000);            conn.setRequestProperty("Accept-Charset", "utf-8");            conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");            conn.setInstanceFollowRedirects(true);            conn.getResponseCode();// trigger server redirect            realURL = conn.getURL().toString();//            Log.d("TAG", str + "\r\n" + "redirect to \r\n" + realURL);        }catch (Exception e) {            e.printStackTrace();        } finally {            if (conn != null) {                conn.disconnect();            }        }        return realURL;    }    /**     * 方法完全自己接管重定向过程，直接调用HttpURLConnection的setInstanceFollowRedirects(false)，     * 传入参数为false，然后递归的方式进行http请求，然后从header参数中取出location字段。     * 看看转了多少次     *     * */    public String recursiveTraceGetPath(String path) {        String realURL = null;        if (mStackDeep.getAndDecrement() > 0) {// 避免异常递归导致StackOverflowError            URL url = null;            HttpURLConnection conn = null;            try {                url = new URL(path);                conn = (HttpURLConnection) url.openConnection();                conn.setRequestProperty("Accept-Charset", "utf-8");                conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");                conn.setConnectTimeout(30000);                conn.setReadTimeout(30000);                conn.setInstanceFollowRedirects(false);                int code = conn.getResponseCode();// Network block   301,302                if (needRedirect(code)) {                    //临时重定向和永久重定向location的大小写有区分                    String location = conn.getHeaderField("Location");                    if (location == null) {                        location = conn.getHeaderField("location");                    }                    if (!(location.startsWith("http://") || location                            .startsWith("https://"))) {                        //某些时候会省略host，只返回后面的path，所以需要补全url                        URL origionUrl = new URL(path);                        location = origionUrl.getProtocol() + "://"                                + origionUrl.getHost() + location;                    }//                    Log.d("TAG", "redirect to \r\n" + location);                    return recursiveTraceGetPath(location);                } else {                    // redirect finish.                    realURL = path;                }            } catch (MalformedURLException e) {                Log.w("TAG", "recursiveTracePath MalformedURLException");            } catch (IOException e) {                Log.w("TAG", "recursiveTracePath IOException");            } catch (Exception e) {                Log.w("TAG", "unknow exception");            } finally {                if (conn != null) {                    conn.disconnect();                }            }        }        return realURL;    }    private boolean needRedirect(int code) {        return (code == HttpStatus.SC_MOVED_PERMANENTLY                || code == HttpStatus.SC_MOVED_TEMPORARILY                || code == HttpStatus.SC_SEE_OTHER || code == HttpStatus.SC_TEMPORARY_REDIRECT);    }

自己写的方法可以自定义重定向的次数，但是需要一个jar文件的支持    import org.apache.http.HttpStatus;

但是如果网页不是重定向，而是使用js 跳转，我写网页也用过这个，用来用户登录后获取首页显示的数据，比如一个简单的

  <body> <span style="white-space:pre"></span><script>var jump_url="http://active.cliim.cn/AJ8nLm?e1baa1462641130bb5295595b173123b2237fe5268";          window.location.href=jump_url;          // setTimeout(function(){          // window.location.href=jump_url;          // }, 500 );  </script> </body>

那我们就真的需要读取网页数据了。使用StringBuffer  tempHTML 存储起来，百度一个html 在线格式化的，格式一下获取的数据，方便阅读

public void doGetURL(String url) throws Exception {        url = redirectGetPath(url);//        url = recursiveTraceGetPath(url);        URL localURL = new URL(url);        URLConnection connection = localURL.openConnection();        HttpURLConnection httpURLConnection = (HttpURLConnection)connection;        httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");        httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");        httpURLConnection.setConnectTimeout(30000);        httpURLConnection.setReadTimeout(30000);        if (httpURLConnection.getResponseCode() >= 300) {            throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());        }        String realURL = httpURLConnection.getURL().toString();        Log.d("TAG", url + "\r\n" + "redirect to \r\n" + realURL);        inputStream = httpURLConnection.getInputStream();        inputStreamReader = new InputStreamReader(inputStream);        reader = new BufferedReader(inputStreamReader);        while ((tempLine = reader.readLine()) != null) {            tempHTML.append(tempLine);        }    }

剩下的就是对获取的网页源码的提取了，常用的就是字符串的一些操作，比如字符串截取，字符定位，

<span style="white-space:pre"></span>tempHTML.substring(tempHTML.indexOf("***"),tempHTML.lastIndexOf("***") + 4);

获取相要的结果在listview 里显示

protected void onPostExecute(StringBuffer result) {        if(result != null){//            ArrayList ls =  new ArrayList<String>(){ { add("str01");  add("str02"); } };            ArrayList ls =  new ArrayList<String>();            ls.add("ClientVersion: android 1.1.1");            ls.add("获取url网页数据");            doHtmlGetRealUrl(result, ls);            for(int i = 0; i<ls.size();i++) {                helper.getReadableDatabase().execSQL("insert into url_table values(null, ?)",                        new String[]{ls.get(i).toString()});            }            listview.setAdapter(new ArrayAdapter<String>(context, android.R.layout.simple_list_item_1, ls));        }        super.onPostExecute(result);    }

上面的循环里，除了把数据添加在list里，还循环插入了数据库，先新建一个类 UrlInfoDatabaseHelper  extends  SQLiteOpenHelper

我只是简单存储一下，就只有一个id，一个content

public class UrlInfoDatabaseHelper extends SQLiteOpenHelper {    final String SQL_CREATE_TABLE = "create table url_table (" +            "_id integer primary key autoincrement, " +            "url_content varchar(1000))";    /*     * 构造方法 :     * 参数介绍 :     * 参数① : 上下文对象     * 参数② : 数据库名称     * 参数③ : 数据库版本号     */    public UrlInfoDatabaseHelper(Context context, String name, int version) {        super(context, name, null, version);    }    @Override    public void onCreate(SQLiteDatabase db) {        db.execSQL(SQL_CREATE_TABLE);    }    @Override    public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {        System.out.println("call update");    }}

初始化时

//数据库帮助类    private UrlInfoDatabaseHelper helper;        helper = new UrlInfoDatabaseHelper(context, "url", 1);

然后怎么从数据库里查出来呢

 private void showSQlDate(){        Cursor cursor = helper.getReadableDatabase().rawQuery("select * from url_table", null);        //遍历Cursor//        while(cursor.moveToNext()){//            Log.e("TAG",cursor.getString(1));//        }        inflateListView(cursor);    }    /*    * 刷新数据库列表显示    * 1. 关联SimpleCursorAdapter与数据库表, 获取数据库表中的最新数据    * 2. 将最新的SimpleCursorAdapter设置给ListView    */    private void inflateListView(Cursor cursor) {        SimpleCursorAdapter cursorAdapter = new SimpleCursorAdapter(                getApplicationContext(),//上下文对象                R.layout.item,  //List显示布局                cursor,//数据源                new String[]{"url_content"}, //游标数据的名称，实际是Table列名字                new int[]{R.id.textView});//填充到的布局文件        listView.setAdapter(cursorAdapter);    }

这里是把数据库的数据在listview 里显示 ，直接使用 SimpleCursorAdapter 

在数据库显示页面，我做了一个新增和删除

新增是显示一个dialog 

private void add(){        //步骤2.1：通过LayoutInflater从Android的XML文件中生成View        LayoutInflater inflater = LayoutInflater.from(this);        final View addView = inflater.inflate(R.layout.add_dialgo,null);        //步骤2.2：通过AlertDialog弹出对话框，并且在第一个button，即PositiveButton监听事件，触发操作        new AlertDialog.Builder(this)                .setTitle("添加框")                .setView(addView)                .setPositiveButton("确定", new DialogInterface.OnClickListener() {                    //我们希望得到addView中的数据，但是这个inner class，只能获取final的值，所以之前将addView设置为final，也就是所有addView的地址是固定的，而不是动态生成。                    public void onClick(DialogInterface dialog, int which) {                        EditText nameView = (EditText)addView.findViewById(R.id.show_name);                        // addData是下面步骤三，实现SQLite的数据更新和ListView的显示同步add(name,weight);                        addData(nameView.getText().toString());                    }                })                .setNegativeButton("取消",null)                .show();    }   // 更新数据库和同步ListView，具体如下：    private void addData(String name){        /* 略去数据的判断，例如如果name一样，采用update的方式等等*/        //步骤3.1 在数据库表格中添加数据        helper.getReadableDatabase().execSQL("insert into url_table values(null, ?)",                new String[]{name});        //步骤3.2 同步ListView，更新游标的信息        showSQlDate();    }

删除时，是用户长按屏幕上的某条数据，使用ContextMenu，即用户手指长按某个View触发的菜单

   private static final int DELETE_ID = 2;    @Override    protected void onCreate(Bundle savedInstanceState) {        super.onCreate(savedInstanceState);        setContentView(R.layout.content_sql);        helper = new UrlInfoDatabaseHelper(this, "url", 1);        listView = (ListView) findViewById(R.id.listViewSql);        /**         * ContextMenu用户手指长按某个View触发的菜单         * 实现场景：用户长按某个List元素，则弹出ContextMenu，选择菜单“Delete”，按下后，弹出AlertDialog，         * 请用户再去确定是否删除，确定后将数据从SQLite中删除，并更新ListView的显示。         * */        //向ListView注册Context Menu，当系统检测到用户长按某单元是，触发Context Menu弹出        registerForContextMenu(listView);        showSQlDate();        findViewById(R.id.btn_sqlAdd).setOnClickListener(new View.OnClickListener() {            @Override            public void onClick(View arg0) {                add();            }        });    }    // 步骤2：创建ContextMenu同OptionMenu，用户长按元素后，会弹出菜单    public void onCreateContextMenu(ContextMenu menu, View v,  ContextMenu.ContextMenuInfo menuInfo) {        menu.add(Menu.NONE, DELETE_ID , Menu.NONE, "Delete");        super.onCreateContextMenu(menu, v, menuInfo);    }    //步骤 3: ContextMenu的触发操作，例子将触发delete() 还可以做编辑等待    public boolean onContextItemSelected(MenuItem item) {        switch(item.getItemId()){            case DELETE_ID:            /* 在此处，我们关键引入 AdapterView.AdapterContextMenuInfo来获取单元的信息。                在有三个重要的信息。                 1、id：The row id of the item for which the context menu is being displayed ，                    在cursorAdaptor中，实际就是表格的_id序号；                 2、position 是list的元素的顺序；                 3、view就可以获得list中点击元素的View， 通过view可以获取里面的显示的信息              */                AdapterView.AdapterContextMenuInfo info = (AdapterView.AdapterContextMenuInfo)item.getMenuInfo();                delete(info.id);                return true;            default:                break;        }        return super.onContextItemSelected(item);    }    //步骤4: 对触发弹框，和Add的相似，确定后，更新数据库和更新ListView的显示，上次学习已有相类的例子，不再重复。其中getNameById是通过id查名字的方法。值得注意的是，为了内部类中使用，delete的参数采用来final的形式。    private void delete(final long  rowId){        if(rowId>0){            new AlertDialog.Builder(this)                    .setTitle("确定要删除?")                    .setPositiveButton("确定", new DialogInterface.OnClickListener() {                        public void onClick(DialogInterface dialog, int which) {                            deleteData(rowId);                        }                    })                    .setNegativeButton("取消", null)                    .show();        }    }    private void deleteData(long rowId){        String[] str = {String.valueOf(rowId)};        helper.getReadableDatabase().delete("url_table", "_id=?", str);        showSQlDate();    }

附件：

源码下载：http://download.csdn.net/detail/i_do_can/9409977