Question

我写了一个shell脚本，它读取一些文件并将这些文件中的所有数据复制到其他合并文件中。

这种情况会多次发生，以前复制的数据不会再次复制。为此，我将最后复制的行保存在单独的文件中，并从第二次比较它。

逻辑流程：

首次阅读单个文件，并将除最后一行以外的整个数据复制到合并文件中。将最后复制的行保存到临时文件中。

sed '$d' $file >> $consolidatedCSVFile

tail -2 $file | head -1 > $consolidatedCSVFile$lastLines/$(basename $file)$lastLine

从第二次开始，从临时文件中读取最后一行，并在单个文件中搜索该行。找到后，从下一行获取所有行并附加到合并文件。

完整脚本：

#!/bin/bash

consolidatedCSVFile=$1
endTime=$2

curdate=$(date +%s)

CSVFiles=${@:3}
CSVFiles=${CSVFiles[@]}

lastLines=_LASTLINES
lastLine=_LASTLINE

newMerge=false

# Returns 1 if Consolidated file is empty
[ -s $consolidatedCSVFile  ]
if [ $? == 1  ]
    then
        mkdir $consolidatedCSVFile$lastLines
        newMerge=true
fi

testInProcess=false

if [ $endTime -ge $curdate  ]
    then
        testInProcess=true
fi

for file in $CSVFiles
do
if $newMerge ;
    then
        touch $consolidatedCSVFile$1astLines5(basename $file)$lastLine
        sed $d $file >> $consolidatedCSVFile
        tail -2 $file | head -1 >   $consolidatedCSVFile$1astLines5(basename $file)$lastLine
    else
        availableLastLine="`cat $consolidatedCSVFile$1astLines/$(basename $file)$lastLine`"
        if $testInProcess
            then
                sed '1,/^'"$availableLastLine"'$/d' $file | sed '$d' >> $consolidatedCSVFile
                tail -2 $file | head -1 > $consolidatedCSVFile$1astLines5(basename $file)$lastLine
            else
                sed '1,/^'"$availableLastLine"'$/d' $file >> $consolidatedCSVFile
        fi
fi

done

if  !  $testInProcess ;
then
    rm -rf $consolidatedCSVFile$lastLines
fi

有时，我会收到错误sed: -e expression #1, char 26: unknown command: 'X'

我的猜测

我正在尝试合并JMeter生成的文件。有时，文件生成的数据类似1439459065780,5,SOAP/XML-RPC Request,200,OK,ws test 1-3,text,true,267,3,3,5

我认为问题出在sed '1,/^'"$availableLastLine"'$/d' $file | sed '$d' >> $consolidatedCSVFile SOAP/XML时。斜线（/）在这里可能有问题。

Answer 1

如果没有完成所有代码，你对这条线的直觉似乎就在我身上。该变量将在执行sed命令之前进行扩展，因此如果扩展中有/，它将终止sed中的模式。您可以使用除/之外的其他字符作为分隔符，因此，如果_不会显示在您的数据中，则可以使用

sed '1,_^'"$availableLastLine"'$_d' $file | sed '$d' >> $consolidatedCSVFile

或者你可以跳过第二个sed

sed -e '1,_^'"$availableLastLine"'$_d' -e '$d' $file >> $consolidatedCSVFile

Answer 2

修改availableLastLine后

工作到availableLastLine=$(sed 's/\//\\\//' $consolidatedCSVFile$lastLines/$(basename $file)$lastLine)。

工作脚本：

#!/bin/bash

consolidatedCSVFile=$1
endTime=$2

curdate=$(date +%s)

CSVFiles=${@:3}
CSVFiles=${CSVFiles[@]}

lastLines=_LASTLINES
lastLine=_LASTLINE

newMerge=false

# Returns 1 if Consolidated file is empty
[ -s $consolidatedCSVFile  ]
    if [ $? == 1  ]
        then
            mkdir $consolidatedCSVFile$lastLines
            newMerge=true
    fi

testInProcess=false

if [ $endTime -ge $curdate  ]
    then
        testInProcess=true
fi

for file in $CSVFiles
do
if $newMerge ;
    then
        touch $consolidatedCSVFile$1astLines5(basename $file)$lastLine
        sed $d $file >> $consolidatedCSVFile
        tail -2 $file | head -1 > $consolidatedCSVFile$1astLines5(basename $file)$lastLine
    else
        ## Replaced below line
        #availableLastLine="`cat $consolidatedCSVFile$1astLines/$(basename $file)$lastLine`"
        availableLastLine=$(sed 's/\//\\\//' $consolidatedCSVFile$lastLines/$(basename $file)$lastLine)
        if $testInProcess
            then
                sed '1,/^'"$availableLastLine"'$/d' $file | sed '$d' >> $consolidatedCSVFile
                tail -2 $file | head -1 > $consolidatedCSVFile$1astLines5(basename $file)$lastLine
            else
                sed '1,/^'"$availableLastLine"'$/d' $file >> $consolidatedCSVFile
        fi
fi

done

if  !  $testInProcess ;
then
    rm -rf $consolidatedCSVFile$lastLines
fi

合并多个文件时可能出现的错误案例

#!/bin/bash

echo -e "\n******* $(date) *******"

echo "Arguments :: $@"

consolidatedCSVFile=$1

echo "consolidatedCSVFile :: $1"

endTime=$2
curdate=$(date +%s)

echo -e "endTime :: $2\n"

CSVFiles=${@:3}
CSVFiles=${CSVFiles[@]}

echo -e "Individual Files : ${CSVFiles[@]}\n"

lastLines=_LASTLINES
lastLine=_LASTLINE

newMerge=false

# Returns 1 if Consolidated file is empty
[ -s $consolidatedCSVFile ]
    if [ $? == 1 ]
        then
        newMerge=true
    fi

echo "newMerge? :: $newMerge"

if $newMerge ;
    then
        if [ ! -d $consolidatedCSVFile$lastLines ]
            then
                echo "creating LASTLINES directory"
                mkdir $consolidatedCSVFile$lastLines
        fi
fi

testInProcess=false

if [ $endTime -ge $curdate ]
    then
    testInProcess=true  
fi

echo -e "testInProcess? :: $testInProcess\n"

for file in $CSVFiles
do

echo -e "\nCurrent file in loop :: $(basename $file)"

if $newMerge ;
    then    
        # Returns 1 if file is empty
        [ -s $file ]
            if [ $? == 1 ]
                then
                    echo "File empty !!"
                            continue
            fi
        echo "creating LASTLINE file"
        touch $consolidatedCSVFile$lastLines/$(basename $file)$lastLine
        #sed '$d' $file >> $consolidatedCSVFile
        sed '1d' $file | sed '$d' >> $consolidatedCSVFile
        echo "Last line to save :: $(tail -1 $consolidatedCSVFile)" 
        tail -1 $consolidatedCSVFile > $consolidatedCSVFile$lastLines/$(basename $file)$lastLine
    else
        # Returns 1 if file is empty
        [ -s $file ]
            if [ $? == 1 ]
                then
                    echo "File empty !!"
                            continue
            fi

        if [ ! -f $consolidatedCSVFile$lastLines/$(basename $file)$lastLine ]
            then
                echo "creating LASTLINE file"
                touch $consolidatedCSVFile$lastLines/$(basename $file)$lastLine
                #sed '$d' $file >> $consolidatedCSVFile
                sed '1d' $file | sed '$d' >> $consolidatedCSVFile
                echo "Last line to save :: $(tail -1 $consolidatedCSVFile)" 
                tail -1 $consolidatedCSVFile > $consolidatedCSVFile$lastLines/$(basename $file)$lastLine
                continue
            else
                availableLastLine=$(sed 's/\//\\\//' $consolidatedCSVFile$lastLines/$(basename $file)$lastLine)
                echo "Last line from file :: $availableLastLine"
        fi
        if $testInProcess
            then
                tempVar=`sed '1,/^'"$availableLastLine"'$/d' $file | sed '$d'`
                #echo -e "\ntempVar ::\n$tempVar\n"
                    [ -z "$tempVar" ]
                    if [ $? == 1 ]
                        then
                            #sed '1,/^'"$availableLastLine"'$/d' $file | sed '$d' >> $consolidatedCSVFile
                            echo -e "$tempVar" >> $consolidatedCSVFile
                            echo "Last line to save :: $(tail -1 $consolidatedCSVFile)"
                            tail -1 $consolidatedCSVFile > $consolidatedCSVFile$lastLines/$(basename $file)$lastLine
                        else
                            echo "No last line to save"
                    fi
            else
                        sed '1,/^'"$availableLastLine"'$/d' $file >> $consolidatedCSVFile
        fi
fi

done

if ! $testInProcess ;
then
    echo -e "\nRemoving LASTLINES folder"
    rm -rf $consolidatedCSVFile$lastLines
fi

echo -e "\n*************"

sed throw sed：-e expression＃（number），char（number）：unknown命令：`（letter）'

2 个答案: