Linux下如何使用awk解析json数据

近期在做一个项目，调用api后返回了一个嵌套较多层的json格式的数据，常规解析json的方式也已经很难或做不到提取出需要的信息点，恰好对awk比较熟悉，考虑到awk本身也是一种用于数据处理的工具，于是就有了以下用awk解析json的尝试。下面以face++ 的detect api返回值，json格式：

{  "faces": [    {      "face_token": "826536dba3973d71d219a15e18c9597b",      "face_rectangle": {        "height": 113,        "left": 198,        "top": 125,        "width": 113      },      "attributes": {        "age": {          "value": 26        },        "gender": {          "value": "Female"        }      },      "landmark": {        "mouth_upper_lip_bottom": {          "x": 257,          "y": 203        },        "right_eye_upper_right_quarter": {          "x": 284,          "y": 142        },        "left_eyebrow_upper_right_quarter": {          "x": 235,          "y": 127        },        "mouth_upper_lip_right_contour3": {          "x": 267,          "y": 202        },        "left_eye_top": {          "x": 228,          "y": 143        },        "nose_contour_right3": {          "x": 265,          "y": 185        },        "contour_right6": {          "x": 298,          "y": 207        },        "mouth_lower_lip_bottom": {          "x": 257,          "y": 211        },        "nose_contour_right2": {          "x": 268,          "y": 172        },        "nose_contour_right1": {          "x": 264,          "y": 149        },        "nose_contour_left3": {          "x": 250,          "y": 185        },        "nose_contour_left2": {          "x": 246,          "y": 172        },        "nose_contour_left1": {          "x": 249,          "y": 150        },        "right_eyebrow_upper_right_quarter": {          "x": 286,          "y": 128        },        "left_eyebrow_upper_middle": {          "x": 226,          "y": 126        },        "right_eye_upper_left_quarter": {          "x": 273,          "y": 143        },        "right_eyebrow_right_corner": {          "x": 292,          "y": 133        },        "contour_right8": {          "x": 283,          "y": 227        },        "right_eyebrow_upper_left_quarter": {          "x": 271,          "y": 127        },        "contour_right2": {          "x": 309,          "y": 159        },        "contour_right3": {          "x": 309,          "y": 172        },        "contour_right1": {          "x": 309,          "y": 147        },        "left_eye_lower_left_quarter": {          "x": 222,          "y": 149        },        "contour_left1": {          "x": 201,          "y": 150        },        "contour_left2": {          "x": 201,          "y": 162        },        "contour_left3": {          "x": 203,          "y": 175        },        "contour_left4": {          "x": 205,          "y": 187        },        "contour_left5": {          "x": 209,          "y": 199        },        "contour_left6": {          "x": 215,          "y": 210        },        "contour_left7": {          "x": 224,          "y": 220        },        "right_eye_bottom": {          "x": 280,          "y": 149        },        "right_eye_lower_right_quarter": {          "x": 284,          "y": 148        },        "right_eyebrow_left_corner": {          "x": 266,          "y": 132        },        "right_eyebrow_upper_middle": {          "x": 279,          "y": 126        },        "mouth_lower_lip_top": {          "x": 257,          "y": 205        },        "right_eyebrow_lower_left_quarter": {          "x": 272,          "y": 132        },        "left_eyebrow_lower_left_quarter": {          "x": 219,          "y": 133        },        "mouth_upper_lip_left_contour3": {          "x": 247,          "y": 203        },        "left_eyebrow_lower_middle": {          "x": 226,          "y": 132        },        "left_eye_upper_left_quarter": {          "x": 222,          "y": 144        },        "mouth_upper_lip_left_contour1": {          "x": 252,          "y": 198        },        "mouth_upper_lip_top": {          "x": 257,          "y": 199        },        "mouth_upper_lip_left_contour2": {          "x": 244,          "y": 200        },        "right_eye_pupil": {          "x": 279,          "y": 145        },        "mouth_lower_lip_right_contour1": {          "x": 267,          "y": 204        },        "mouth_lower_lip_left_contour2": {          "x": 242,          "y": 207        },        "mouth_lower_lip_right_contour3": {          "x": 265,          "y": 210        },        "mouth_lower_lip_right_contour2": {          "x": 272,          "y": 206        },        "contour_chin": {          "x": 259,          "y": 237        },        "contour_left9": {          "x": 245,          "y": 235        },        "left_eye_lower_right_quarter": {          "x": 233,          "y": 150        },        "mouth_left_corner": {          "x": 236,          "y": 202        },        "contour_right4": {          "x": 307,          "y": 184        },        "contour_right7": {          "x": 291,          "y": 217        },        "left_eyebrow_left_corner": {          "x": 211,          "y": 135        },        "nose_right": {          "x": 272,          "y": 181        },        "nose_tip": {          "x": 257,          "y": 177        },        "contour_right5": {          "x": 303,          "y": 196        },        "nose_contour_lower_middle": {          "x": 258,          "y": 187        },        "right_eye_top": {          "x": 278,          "y": 141        },        "mouth_lower_lip_left_contour3": {          "x": 249,          "y": 210        },        "right_eye_right_corner": {          "x": 288,          "y": 146        },        "mouth_upper_lip_right_contour1": {          "x": 261,          "y": 198        },        "mouth_upper_lip_right_contour2": {          "x": 270,          "y": 199        },        "right_eyebrow_lower_right_quarter": {          "x": 285,          "y": 132        },        "left_eye_left_corner": {          "x": 218,          "y": 148        },        "mouth_right_corner": {          "x": 278,          "y": 201        },        "right_eye_lower_left_quarter": {          "x": 275,          "y": 149        },        "left_eyebrow_right_corner": {          "x": 242,          "y": 132        },        "left_eyebrow_lower_right_quarter": {          "x": 234,          "y": 132        },        "right_eye_center": {          "x": 279,          "y": 146        },        "left_eye_pupil": {          "x": 228,          "y": 146        },        "nose_left": {          "x": 243,          "y": 182        },        "mouth_lower_lip_left_contour1": {          "x": 247,          "y": 204        },        "left_eye_upper_right_quarter": {          "x": 233,          "y": 145        },        "right_eyebrow_lower_middle": {          "x": 279,          "y": 132        },        "left_eye_center": {          "x": 228,          "y": 148        },        "contour_left8": {          "x": 233,          "y": 228        },        "contour_right9": {          "x": 272,          "y": 234        },        "right_eye_left_corner": {          "x": 270,          "y": 149        },        "left_eyebrow_upper_left_quarter": {          "x": 218,          "y": 128        },        "left_eye_bottom": {          "x": 227,          "y": 150        },        "left_eye_right_corner": {          "x": 237,          "y": 149        }      }    }  ],  "time_used": 251,  "request_id": "1492784536,22b21204-0e1e-4071-91ba-b29e5a68fe03",  "image_id": "6kox5v0GjhI9f+MWEc3wIA=="}

我们从调用api开始讲起，以下是我对face++的一个调用，直接运用shell命令的curl发送的post请求：

curl -X POST "https://api-cn.faceplusplus.com/facepp/v3/detect" \        -F "api_key=RF_13WYz9Cgz9KXtJxaqAIbIF7_jxuC-" \        -F "api_secret=MeCM2hq-SSiWXgxMt-JcpOo-t5IAIdz4" \        -F "image_file=@/home/gec/timg.jpeg" \        -F "return_landmark=1" -F "return_attributes=gender,age"

为了方便接下来awk对数据的解析，我们可以将调用api的返回值重定向到一个文件里，将以上命令改写成：

curl -X POST "https://api-cn.faceplusplus.com/facepp/v3/detect" \        -F "api_key=RF_13WYz9Cgz9KXtJxaqAIbIF7_jxuC-" \        -F "api_secret=MeCM2hq-SSiWXgxMt-JcpOo-t5IAIdz4" \        -F "image_file=@/home/gec/timg.jpeg" \        -F "return_landmark=1" -F "return_attributes=gender,age" >face

如此，当调用成功时，我们就可以在face中看到face++的detect api的正确返回值，查看文件用cat指令或者直接用文本编辑器打开都可以，如下：

cat face

{"image_id": "6kox5v0GjhI9f+MWEc3wIA==", "request_id": "1492784536,22b21204-0e1e-4071-91ba-b29e5a68fe03", "time_used": 251, "faces": [{"landmark": {"mouth_upper_lip_left_contour2": {"y": 200, "x": 244}, "mouth_upper_lip_top": {"y": 199, "x": 257}, "mouth_upper_lip_left_contour1": {"y": 198, "x": 252}, "left_eye_upper_left_quarter": {"y": 144, "x": 222}, "left_eyebrow_lower_middle": {"y": 132, "x": 226}, "mouth_upper_lip_left_contour3": {"y": 203, "x": 247}, "left_eyebrow_lower_left_quarter": {"y": 133, "x": 219}, "right_eyebrow_lower_left_quarter": {"y": 132, "x": 272}, "right_eye_pupil": {"y": 145, "x": 279}, "mouth_lower_lip_right_contour1": {"y": 204, "x": 267}, "mouth_lower_lip_left_contour2": {"y": 207, "x": 242}, "mouth_lower_lip_right_contour3": {"y": 210, "x": 265}, "mouth_lower_lip_right_contour2": {"y": 206, "x": 272}, "contour_chin": {"y": 237, "x": 259}, "contour_left9": {"y": 235, "x": 245}, "left_eye_lower_right_quarter": {"y": 150, "x": 233}, "mouth_lower_lip_top": {"y": 205, "x": 257}, "right_eyebrow_upper_middle": {"y": 126, "x": 279}, "right_eyebrow_left_corner": {"y": 132, "x": 266}, "right_eye_lower_right_quarter": {"y": 148, "x": 284}, "right_eye_bottom": {"y": 149, "x": 280}, "contour_left7": {"y": 220, "x": 224}, "contour_left6": {"y": 210, "x": 215}, "contour_left5": {"y": 199, "x": 209}, "contour_left4": {"y": 187, "x": 205}, "contour_left3": {"y": 175, "x": 203}, "contour_left2": {"y": 162, "x": 201}, "contour_left1": {"y": 150, "x": 201}, "left_eye_lower_left_quarter": {"y": 149, "x": 222}, "contour_right1": {"y": 147, "x": 309}, "contour_right3": {"y": 172, "x": 309}, "contour_right2": {"y": 159, "x": 309}, "mouth_left_corner": {"y": 202, "x": 236}, "contour_right4": {"y": 184, "x": 307}, "contour_right7": {"y": 217, "x": 291}, "left_eyebrow_left_corner": {"y": 135, "x": 211}, "nose_right": {"y": 181, "x": 272}, "nose_tip": {"y": 177, "x": 257}, "contour_right5": {"y": 196, "x": 303}, "nose_contour_lower_middle": {"y": 187, "x": 258}, "right_eye_top": {"y": 141, "x": 278}, "mouth_lower_lip_left_contour3": {"y": 210, "x": 249}, "right_eye_right_corner": {"y": 146, "x": 288}, "mouth_upper_lip_right_contour1": {"y": 198, "x": 261}, "mouth_upper_lip_right_contour2": {"y": 199, "x": 270}, "right_eyebrow_lower_right_quarter": {"y": 132, "x": 285}, "left_eye_left_corner": {"y": 148, "x": 218}, "mouth_right_corner": {"y": 201, "x": 278}, "right_eye_lower_left_quarter": {"y": 149, "x": 275}, "left_eyebrow_right_corner": {"y": 132, "x": 242}, "left_eyebrow_lower_right_quarter": {"y": 132, "x": 234}, "right_eye_center": {"y": 146, "x": 279}, "left_eye_pupil": {"y": 146, "x": 228}, "nose_left": {"y": 182, "x": 243}, "mouth_lower_lip_left_contour1": {"y": 204, "x": 247}, "left_eye_upper_right_quarter": {"y": 145, "x": 233}, "right_eyebrow_lower_middle": {"y": 132, "x": 279}, "left_eye_center": {"y": 148, "x": 228}, "contour_left8": {"y": 228, "x": 233}, "contour_right9": {"y": 234, "x": 272}, "right_eye_left_corner": {"y": 149, "x": 270}, "left_eyebrow_upper_left_quarter": {"y": 128, "x": 218}, "left_eye_bottom": {"y": 150, "x": 227}, "left_eye_right_corner": {"y": 149, "x": 237}, "right_eyebrow_upper_left_quarter": {"y": 127, "x": 271}, "contour_right8": {"y": 227, "x": 283}, "right_eyebrow_right_corner": {"y": 133, "x": 292}, "right_eye_upper_left_quarter": {"y": 143, "x": 273}, "left_eyebrow_upper_middle": {"y": 126, "x": 226}, "right_eyebrow_upper_right_quarter": {"y": 128, "x": 286}, "nose_contour_left1": {"y": 150, "x": 249}, "nose_contour_left2": {"y": 172, "x": 246}, "nose_contour_left3": {"y": 185, "x": 250}, "nose_contour_right1": {"y": 149, "x": 264}, "nose_contour_right2": {"y": 172, "x": 268}, "mouth_lower_lip_bottom": {"y": 211, "x": 257}, "contour_right6": {"y": 207, "x": 298}, "nose_contour_right3": {"y": 185, "x": 265}, "left_eye_top": {"y": 143, "x": 228}, "mouth_upper_lip_right_contour3": {"y": 202, "x": 267}, "left_eyebrow_upper_right_quarter": {"y": 127, "x": 235}, "right_eye_upper_right_quarter": {"y": 142, "x": 284}, "mouth_upper_lip_bottom": {"y": 203, "x": 257}}, "attributes": {"gender": {"value": "Female"}, "age": {"value": 26}}, "face_rectangle": {"width": 113, "top": 125, "left": 198, "height": 113}, "face_token": "826536dba3973d71d219a15e18c9597b"}]}

原本json结构是相当清晰的，每条数据都会按行分开，但是经实验Linux下（也可能不是环境原因，这里不是很清楚，还请朋友们不吝指正）返回的json是不分行的，所有数据都挤在一行上，这对解析来说无疑又加大了难度。

接下来尝试从json当中提取需要的信息，以年龄age为例。如果信息太多觉得不好找，可以用grep指令标记一下，这样就可以很方便的找到关键词的所在位置：

grep age face

博主最初的思路是：awk能不能通过匹配到age这个词，然后输出当前位置若干字符后的字符串26？博主按这个思路尝试了很久，同时也发现网上有很多朋友问过类似的问题，但是都得不到答案，大概awk本身并没有能直接实现这种需求的语法吧。

我最终想出来的方法是awk指定分隔符结合管道运用，指定 所需数据附近的符号 作为分隔符将原json数据切块，越切越小，切到最后保留的那块数据就是我们想要的数据。

这个方法的优点在于无视了json格式的多层嵌套进行了直接切块，完整指令如下：

awk -F '"' '{print $(NF-14)}' face|awk -F} '{print $1}'|awk -F:' ' '{print $2}'

将上面的指令剖析开来

第一步：

awk -F '"' '{print $(NF-14)}' face         #指定引号为分隔符，打印倒数第14个字段

得到：

: 26}}, 

第二步，基于上面的结果再进行：

awk -F} '{print $1}'                        #指定｝为分隔符，打印第1个字段

得到：

: 26

第三步，基于上面结果再进行：

awk -F:' ' '{print $2}'                     #指定：和空格 为分隔符，打印第2个字段

得到结果：

26

至此，awk顺利提取得到了json格式中的数据。由于json格式都是差不多的形式，指定分隔符加管道的方法同样适用于其他api返回数据的解析，仅需修改字段数即可。

另外，linux下使用jq也是一个不错的选择：linux下使用jq解析json