# 调用命令
[root@iZ2ze7889ommtwxghsyd0iZ]/home/ops/vgdog/suricata_2# scripts/setup-app-layer.py --logger --parser --detect IOT iot_header_type
 
# git查看新文件/修改文件，可以看到下面的内容
On branch vgdog/support_iot
Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
	modified:   rust/src/lib.rs
	modified:   src/Makefile.am
	modified:   src/app-layer-parser.c
	modified:   src/app-layer-protos.c
	modified:   src/app-layer-protos.h
	modified:   src/detect-engine-register.h
	modified:   src/output.c
	modified:   suricata.yaml.in

Untracked files:
  (use "git add <file>..." to include in what will be committed)
	rust/src/applayeriot/
	src/detect-iot-iot_header_type.c
	src/detect-iot-iot_header_type.h
	src/output-json-iot.c
	src/output-json-iot.h

no changes added to commit (use "git add" and/or "git commit -a")

[root@iZ2ze7889ommtwxghsyd0iZ]/home/ops/vgdog/suricata_2# ll rust/src/applayeriot
total 28
-rw-r--r-- 1 root root  1347 Jul  9 08:58 logger.rs
-rw-r--r-- 1 root root   807 Jul  9 08:58 mod.rs
-rw-r--r-- 1 root root  2022 Jul  9 08:58 parser.rs
-rw-r--r-- 1 root root 15927 Jul  9 08:58 iot.rs

pub mod iot;
pub mod logger;
// pub mod detect;
mod parser;

pub struct IOTTransaction {
    tx_id: u64,
    
    // pub request: Option<String>,
    // pub response: Option<String>,
    pub message: Option<IOTMessage>,
    tx_data: AppLayerTxData,
}

#[derive(AppLayerEvent)]
enum IOTEvent {
    TooManyTransactions,
    IOTHelloEvent,
    IOTHelloResEvent,
    ...
    IOTAlertEvent,
}



impl IOTState {
    ......

    fn parse(&mut self, input: &[u8]) -> AppLayerResult {
        // We're not interested in empty requests.
        if input.is_empty() {
            return AppLayerResult::ok();
        }

        let mut start = input;
        while !start.is_empty() {
            match parser::parse_message(start) {
                Ok((rem, message)) => {
                    start = rem;

                    // SCLogNotice!("Message: {:?}", message);
                    let mut tx = self.new_tx();
                    if self.transactions.len() >= unsafe { IOT_MAX_TX } {
                        tx.tx_data.set_event(IOTEvent::TooManyTransactions as u8);
                    }
                    match message.header.iot_type.as_ref() {
                        "Hello" => tx.tx_data.set_event(IOTEvent::IOTHelloEvent as u8),
                        ...
                        _ => {
                            SCLogNotice!("Inpossible message header cmd not valid: {:?}", message);
                        },
                    }
                    tx.message = Some(message);

                    self.transactions.push_back(tx);
                    if self.transactions.len() >= unsafe { IOT_MAX_TX } {
                        return AppLayerResult::err();
                    }
                }
                Err(nom::Err::Incomplete(_)) => {
                    // Not enough data. This parser doesn't give us a good indication
                    // of how much data is missing so just ask for one more byte so the
                    // parse is called as soon as more data is received.
                    let consumed = input.len() - start.len();
                    let needed = start.len() + 1;
                    return AppLayerResult::incomplete(consumed as u32, needed as u32);
                }
                Err(_) => {
                    return AppLayerResult::err();
                }
            }
        }

        // Input was fully consumed.
        return AppLayerResult::ok();
    }
    ...
}

/// C entry point for a probing parser.
unsafe extern "C" fn rs_iot_probing_parser(
    _flow: *const Flow, _direction: u8, input: *const u8, input_len: u32, _rdir: *mut u8,
) -> AppProto {
    // SCLogError!("Inside iot probing parser");
    // Need at least 2 bytes.
    if input_len > 1 && !input.is_null() {
        // SCLogError!("Hit parse input");
        let slice = build_slice!(input, input_len as usize);
        if parse_header(slice).is_ok() {
            // SCLogNotice!("Parse iot message success");
            return ALPROTO_IOT;
        }
    }
    return ALPROTO_UNKNOWN;
}


unsafe extern "C" fn rs_iot_tx_get_alstate_progress(tx: *mut c_void, _direction: u8) -> c_int {
    // uni-direction, stateless parser, simply use 1.
    return 1;
}


#[no_mangle]
pub unsafe extern "C" fn rs_iot_register_parser() {
    let default_port = CString::new("[7000]").unwrap();
    let parser = RustParser {
        name: PARSER_NAME.as_ptr() as *const c_char,
        default_port: default_port.as_ptr(),
        ipproto: IPPROTO_TCP,
        probe_ts: Some(rs_iot_probing_parser),
        probe_tc: Some(rs_iot_probing_parser),
        min_depth: 0,
        max_depth: 16,
        state_new: rs_iot_state_new,
        state_free: rs_iot_state_free,
        tx_free: rs_iot_state_tx_free,
        parse_ts: rs_iot_parse,
        parse_tc: rs_iot_parse,
        get_tx_count: rs_iot_state_get_tx_count,
        get_tx: rs_iot_state_get_tx,
        tx_comp_st_ts: 1,
        tx_comp_st_tc: 1,
        tx_get_progress: rs_iot_tx_get_alstate_progress,
        get_eventinfo: Some(IOTEvent::get_event_info),
        get_eventinfo_byid: Some(IOTEvent::get_event_info_by_id),
        localstorage_new: None,
        localstorage_free: None,
        get_tx_files: None,
        get_tx_iterator: Some(applayer::state_get_tx_iterator::<IOTState, IOTTransaction>),
        get_tx_data: rs_iot_get_tx_data,
        get_state_data: rs_iot_get_state_data,
        apply_tx_config: None,
        flags: APP_LAYER_PARSER_OPT_ACCEPT_GAPS,
        truncate: None,
        get_frame_id_by_name: None,
        get_frame_name_by_id: None,
    };

    let ip_proto_str = CString::new("tcp").unwrap();

    if AppLayerProtoDetectConfProtoDetectionEnabled(ip_proto_str.as_ptr(), parser.name) != 0 {
        let alproto = AppLayerRegisterProtocolDetection(&parser, 1);
        ALPROTO_IOT = alproto;
        if AppLayerParserConfParserEnabled(ip_proto_str.as_ptr(), parser.name) != 0 {
            let _ = AppLayerRegisterParser(&parser, alproto);
        }
        if let Some(val) = conf_get("app-layer.protocols.iot.max-tx") {
            if let Ok(v) = val.parse::<usize>() {
                IOT_MAX_TX = v;
            } else {
                SCLogError!("Invalid value for iot.max-tx");
            }
        }
        SCLogNotice!("Rust iot parser registered.");
    } else {
        SCLogNotice!("Protocol detector and parser disabled for IOT.");
    }
}

/// Get the header type buffer for a transaction from C.
///
/// No required for parsing, but an example function for retrieving a
/// pointer to the request buffer from C for detection.
#[no_mangle]
pub unsafe extern "C" fn rs_iot_get_header_type_buffer(
    tx: *mut c_void, buf: *mut *const u8, len: *mut u32,
) -> u8 {
    let tx = cast_pointer!(tx, IOTTransaction);
    if let Some(ref message) = tx.message {
        if !message.header.iot_type.is_empty() {
            *len = message.header.iot_type.len() as u32;
            *buf = message.header.iot_type.as_ptr();
            return 1;
        }
    }
    return 0;
}

fn log_iot(tx: &IOTTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> {
    js.open_object("iot")?;
    if let Some(ref message) = tx.message {
        match message.header.iot_type.as_ref() {
            "Hello" => js.set_string("type", "Connect")?,
            ...
            _ => js.set_string("cmd", "unknown")?
        };
    }
    ...
    js.close()?;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use nom7::Err;
    use std::time::{Instant, Duration};

    #[test]
    fn test_encode_bigendian_u16() {
        let size = 2;
...
        let u16_1024 = 1024;
        let expect_u16_1024_bigendian: [u8; 2] = [0x04, 0x00];
...
        match encode_bigendian_u16(u16_1024) {
            Ok((_, array)) => {
                assert_eq!(&array[..size], &expect_u16_1024_bigendian);
            },
            Err(e) => panic!("Encoding u16 1024 failed: {:?}", e),
        }
    }
}

warning: `suricata` (lib test) generated 27 warnings (run `cargo fix --lib -p suricata --tests` to apply 20 suggestions)
    Finished test [unoptimized + debuginfo] target(s) in 9.78s
     Running unittests src/lib.rs (target/debug/deps/suricata-08d6b4a748e9dd8e)

running 1 test
test applayeriot::parser::tests::test_encode_bigendian_u16 ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 430 filtered out; finished in 0.00s

[root@iZ2ze7889ommtwxghsyd0iZ]/home/ops/vgdog/suricata_2# git status
On branch vgdog/support_iot
Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
        modified:   src/Makefile.am
        modified:   src/app-layer-parser.c
        modified:   src/app-layer-protos.c
        modified:   src/app-layer-protos.h
        modified:   src/detect-engine-register.h
        modified:   src/output.c
        modified:   suricata.yaml.in

Untracked files:
  (use "git add <file>..." to include in what will be committed)
        src/detect-iot-iot_header_type.c
        src/detect-iot-iot_header_type.h
        src/output-json-iot.c
        src/output-json-iot.h

no changes added to commit (use "git add" and/or "git commit -a")

// 注册解析器
void DetectIOTiot_header_typeRegister(void)
{
    sigmatch_table[DETECT_AL_IOT_BUFFER].name = "iot_iot_header_type";
    sigmatch_table[DETECT_AL_IOT_BUFFER].desc =
            "IOT content modifier to match on the iot buffers";
    sigmatch_table[DETECT_AL_IOT_BUFFER].Setup = DetectIOTiot_header_typeSetup;
#ifdef UNITTESTS
    sigmatch_table[DETECT_AL_IOT_BUFFER].RegisterTests = DetectIOTiot_header_typeRegisterTests;
#endif
    sigmatch_table[DETECT_AL_IOT_BUFFER].flags |= SIGMATCH_NOOPT;

    /* register inspect engines */
    DetectAppLayerInspectEngineRegister("iot_buffer", ALPROTO_IOT, SIG_FLAG_TOSERVER, 0,
            DetectEngineInspectIOTiot_header_type, NULL);
    DetectAppLayerInspectEngineRegister("iot_buffer", ALPROTO_IOT, SIG_FLAG_TOCLIENT, 0,
            DetectEngineInspectIOTiot_header_type, NULL);

    g_iot_rust_id = DetectBufferTypeGetByName("iot_buffer");

    SCLogNotice("IOT application layer detect registered.");
}

//真正的解析器
static uint8_t DetectEngineInspectIOTiot_header_type(DetectEngineCtx *de_ctx,
        DetectEngineThreadCtx *det_ctx, const struct DetectEngineAppInspectionEngine_ *engine,
        const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
{
    uint8_t ret = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
    const uint8_t *data = NULL;
    uint32_t data_len = 0;

    if (flags & STREAM_TOSERVER) {
        rs_iot_get_request_buffer(txv, &data, &data_len);
    } else if (flags & STREAM_TOCLIENT) {
        rs_iot_get_response_buffer(txv, &data, &data_len);
    }

    if (data != NULL) {
        const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f,
                data, data_len, 0, DETECT_CI_FLAGS_SINGLE,
                DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
        if (match) {
            ret = DETECT_ENGINE_INSPECT_SIG_MATCH;
        }
    }

    SCLogNotice("Returning %u.", ret);
    return ret;
}

# 好的，这里看来我们新加入的Unit Test正确地注册了
[root@iZ2ze7889ommtwxghsyd0iZ]/home/ops/vgdog/suricata_2# ./src/suricata --list-unittests | grep IOT
Notice: detect-iot-iot_header_type: IOT application layer detect registered. [DetectIOTiot_header_typeRegister:detect-iot-iot_header_type.c:73]
Notice: output-json-iot: IOT JSON logger registered. [JsonIOTLogRegister:output-json-iot.c:170]
DetectIOTiot_header_typeTest

# 好的，看来我们的UT正确执行了
[root@iZ2ze7889ommtwxghsyd0iZ]/home/ops/vgdog/suricata_2# ./src/suricata -u -U DetectIOTiot_header_typeTest
Notice: limesh: Rust limesh parser registered. [suricata::applayerlimesh::limesh::rs_limesh_register_parser:limesh.rs:409]
Notice: iot: Rust iot parser registered. [suricata::applayeriot::iot::rs_iot_register_parser:iot.rs:338]

...

pass
==== TEST RESULTS ====
PASSED: 1
FAILED: 0
======================

SigLoadSignatures {
	...
  SCSigRegisterSignatureOrderingFuncs {
  	# 挂载一串函数到de_ctx->sc_sig_order_funcs，后面挂载的后面用。先挂载OrderByAction，再挂载OrderByPriority
  	SCSigRegisterSignatureOrderingFunc(de_ctx, SCSigOrderByActionCompare);
  	...
  	SCSigRegisterSignatureOrderingFunc(de_ctx, SCSigOrderByPriorityCompare);
  }
  ...
  SCSigOrderSignatures {
  	# 排序，递归比较时使用de_ctx->sc_sig_order_funcs上面注册的顺序判断，用第一个能判断出来大小的比较。都相等的话用SID比较小的ID在前面。最终挂载到检测上下文det_ctx
  	sigw_list = SCSigOrder(sigw_list, de_ctx->sc_sig_order_funcs)
  	...
  }
  ...
  # 真正构建signature组
  # 需了解SGH和MPM的含义，参考https://forum.suricata.io/t/mpm-context-explanation/1463/3
  # SGH理解为Signature HEAD Group，共享pattern的一堆signature
  SigGroupBuild {
  	# 针对每条规则使用RetrieveFPForSig提取single pattern，用于后面构建MPM。这里注意s->init_data->mpm_sm->ctx，这个就是fast pattern。
  	SigPrepareStage1
  	# 按照TCP/UDP/IP做大的分组。开始初步分配sgh
  	SigPrepareStage2
  	# 针对无法按照协议，端口，地址分类构建SGH分组
  	SigPrepareStage3
  	# 在这里真正调用每个detection buffer注册的mpm & spm的函数，挂载prefilter engine。开始在sgh的tx_engine里面挂prefilter engine
  	SigPrepareStage4 {
  		PrefilterSetupRuleGroup {
  			PatternMatchPrepareGroup {
  				# 给已经建好的sgh的pktpayload分配mpm_store结构，包着一个mpm_ctx，准备对per sig附加signature
  				MpmStorePrepareBuffer
  				# 注册pktpayload的mpm
  				PrefilterPktPayloadRegister {
  					# 每个函数
  					PrefilterAppendPayloadEngine
  				}
  				# 该注册applayer的tx mpm了
  				PrepareMpms {
  					for 遍历 DetectBufferMpmRegistry *a = de_ctx->app_mpms_list {
  						# 如果有对应的同样direction呀，协议呀啥的mpm直接利用mpm。把自己的fast pattern加到mpm ctx里面
							MpmStorePrepareBufferAppLayer {
								# 这里开始将每个提取出来的fp添加到de_ctx的mpm store里面，可以理解为往hyperscan的db里面添加patter，存储id
								MpmStoreSetup {
									# 
									for s = de_ctx->sig_array[sig] {
										# 添加fp
										PopulateMpmHelperAddPattern		
									}
								}
								# 把内部mpmstore加到de_ctx->mpm_hash_table
								MpmStoreAdd
							}
							# 这里实际上是spm，单模匹配加函数
              a->PrefilterRegisterWithListId(PrefilterGenericMpmRegister) {
                # 挂载到sgh->init->tx_engines上面
                PrefilterAppendTxEngine {
                  # 分配prefilter引擎，附加到de_ctx的enginelist
                }
              }					
  					}
  				}
  			}
  		}
  	}
  	...
  }
}

DetectAppLayerMpmRegister {
	# 注册PrefilterRegisterWithListId，实际上注册的是PrefilterGenericMpmRegister。PrefilterRegisterWithListId在PrepareMpms里面调用
	# PrefilterGenericMpmRegister在运行时调用PrefilterAppendTxEngine，然后调用PrefilterGenericMpmFree
	
	# 这里挂载buffer id到sm list，用于一会拿fastpattern
	SupportFastPatternForSigMatchList
}

// 参考src/detect-engine-prefilter.c

// 单模式匹配用的是DetectRunPrefilterTx
DetectFlow {
	DetectRun {
		DetectRunTx {
			# 判断下是否发生了一次完整的事务，
			GetDetectTx {
				# 调用prefilterengine来先过滤一批检测，这里实际上调用的是engine->cb.PrefilterTx来处理函数，这里的engine->pectx是一个mpm_ctx
        DetectRunPrefilterTx {
        	# 这里才是真正的执行MPM prefilter的地方
        	engine->cb.PrefilterTx(PrefilterMpm) {
        		# 这里实际上是真正执行search的操作，说白了就是hyperscan的多模式匹配
        		mpm_table[mpm_ctx->mpm_type].Search
        	}
        	# 已经确定了命中的规则后，这里来了一次quicksort排序
        	QuickSortSigIntId
        }
        # 请注意，这里返回的时候实际上是prefilter，还都是所有的规则都有的状态，也就是说即使命中的规则的action是pass，也有返回值
        # 这里开始单条单条匹配，逐个规则判断是不是命中。
        for (uint32_t i = 0; i < array_idx {
        	# 逐条检测命中，如果命中了还要把det_ctx->alert_queue
					DetectRunTxInspectRule{
						# 追加alert，增加alert_queue_size，追加alert到alert_queue
						AlertQueueAppend
        	}	
        }
        	
			}
			...
			# 这里开始整理alert了
			DetectRunPostRules {
        #在这里处理了pass类型的函数
        PacketAlertFinalize {
        	# 先排序，这个排序
    			qsort(det_ctx->alert_queue, det_ctx->alert_queue_size, sizeof(PacketAlert),
            AlertQueueSortHelper);
        }
      }
		}
	}
}