我该如何在golang中优化文件轮换？
我在golang中设计并实现了文件的文件轮换。
根据设计，我根据filesize >= FileSizeThreshold(50000bytes)或file duration >= FileDurationThreshold(1 minute)（以先到者为准）轮换文件。
以下是golang中的实现。
package main
import (
"os"
"path/filepath"
"time"
"log"
"strings"
"flag"
"os/exec"
)
type FileStruct struct{
Filename string
CreatedAt time.Time
}
type FileRotate struct {
Dir string
File chan FileStruct
}
const(
MAX_FILE_SIZE = 50000
MAX_FILE_DURATION = time.Minute * 1
filename_time_format = "20060102150405000"
MAX_TRY = 5
)
var blockingChan chan int
func main(){
path := flag.String("dir", "", "absolute path of dir ")
flag.Parse()
if strings.Contains(*path, "./") {
log.Fatalln("ERROR: please give absolute path")
}
if info, err := os.Stat(*path); err == nil{
if ! info.IsDir(){
log.Fatalln(*path," is not a directory")
}
log.Println("directory found..")
} else {
if os.IsNotExist(err){
log.Println("directory not found..")
log.Println("creating the directory..",*path)
if err := exec.Command("mkdir","-p",*path).Run(); err != nil{
log.Fatalln("failed to create the directory ERROR:",err)
}
log.Println("directory created successfully")
}
}
filerotate := &FileRotate{*path,make(chan FileStruct,1)}
go filerotate.FileOperationsRoutine()
log.Println("generating file name struct..")
filerotate.File <- GetFileStruct()
<- blockingChan
}
func (rotate *FileRotate) FileOperationsRoutine(){
try := 0
var f *os.File
for{
if file, ok := <- rotate.File; ok{
if f == nil {
log.Println("WARN: file ptr is nil")
}
filePath := filepath.Join(rotate.Dir, file.Filename)
fileInfo, err := os.Stat(filePath)
if err != nil && os.IsNotExist(err) {
log.Println("file:", filePath, " does not exist...creating file")
_, err = os.Create(filePath)
if err != nil {
log.Println("failed to create the file ERROR:",err)
try++
if try == MAX_TRY {
log.Println("tried creating the file ",MAX_TRY," times. No luck")
time.Sleep(time.Second * 3)
continue
}
rotate.File <- file
continue
}
log.Println("file:", filePath, " created successfully")
fileInfo,err = os.Stat(filePath)
}
sizeCheck := fileInfo.Size() >= MAX_FILE_SIZE
durationCheck := time.Now().After(file.CreatedAt.Add(MAX_FILE_DURATION))
if sizeCheck || durationCheck {
log.Println("filesize of ",filePath," is ",fileInfo.Size(),"..filesizeCheck=",sizeCheck)
log.Println("fileDurationCheck=",durationCheck)
log.Println("rotating the file..")
f.Close()
f = nil
go ZipAndSendRoutine(filePath)
rotate.File <- GetFileStruct()
}else{
if f == nil {
f, err = os.OpenFile(filePath, os.O_RDWR | os.O_APPEND, 0644)
if err != nil {
log.Println("failed to open the file ERROR:", err)
try++
if try == MAX_TRY {
log.Println("tried opening the file ", MAX_TRY, " times. No luck")
time.Sleep(time.Second * 3)
continue
}
rotate.File <- file
continue
}
log.Println("file opened in append mode")
}
rotate.File <- file
}
}
}
}
func GetFileStruct() FileStruct{
current_time := time.Now()
log.Println("returning the filestruct..")
return FileStruct{"example_" + current_time.Format(filename_time_format),current_time}
}
func ZipAndSendRoutine(file string){
log.Println("zipping and sending the file:",file,"to remote server")
}
执行日志：
root@workstation:/media/sf_golang# ./bin/file_rotation -dir "/tmp/file_rotaion"
2017/01/16 15:05:03 directory found..
2017/01/16 15:05:03 starting file operations routine...
2017/01/16 15:05:03 generating file name struct..
2017/01/16 15:05:03 returning the filestruct..
2017/01/16 15:05:03 WARN: file ptr is nil
2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000 does not exist...creating file
2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000 created successfully
2017/01/16 15:05:03 file opened in append mode
2017/01/16 15:06:03 filesize of /tmp/file_rotaion/example_20170116150503000 is 0 ..filesizeCheck= false ...fileDurationCheck= true
2017/01/16 15:06:03 rotating the file..
2017/01/16 15:06:03 returning the filestruct..
2017/01/16 15:06:03 WARN: file ptr is nil
2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000 does not exist...creating file
2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000 created successfully
2017/01/16 15:06:03 file opened in append mode
2017/01/16 15:06:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150503000 to remote server
2017/01/16 15:07:03 filesize of /tmp/file_rotaion/example_20170116150603000 is 0 ..filesizeCheck= false ...fileDurationCheck= true
2017/01/16 15:07:03 rotating the file..
2017/01/16 15:07:03 returning the filestruct..
2017/01/16 15:07:03 WARN: file ptr is nil
2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000 does not exist...creating file
2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000 created successfully
2017/01/16 15:07:03 file opened in append mode
2017/01/16 15:07:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150603000 to remote server
从日志中可以看出，该实用程序正在按预期工作。 但在执行此实用程序期间，CPU使用率几乎为100％
停止实用程序后..
我已经确定了原因：
FileOperations goroutine无限期运行，在此例程中，我在rotate.File频道上发送文件指针
我陷入困境，不知道如何进一步优化。 谁能告诉我如何优化该实用程序的CPU利用率？
1 个答案:
答案 0 :(得分：3)
您的代码的主要问题是在0向FileStruct或old频道传递{for}循环时。所以没有等待数据的通道接收和if循环内部你正在对文件进行统计以获取其数据，这主要是你必须已经完成的
这是您的计划
new此处大约40秒内有195k系统调用
你可能会做的是为
添加一个等待时间% time seconds usecs/call calls errors syscall
------ ----------- ----------- --------- --------- ----------------
93.58 11.835475 3793 3120 227 futex
6.38 0.807279 4 192048 1 stat
0.03 0.003284 9 366 sched_yield
0.01 0.000759 7 114 rt_sigaction
0.00 0.000271 90 3 openat
0.00 0.000197 10 19 mmap
0.00 0.000143 20 7 write
0.00 0.000071 24 3 clone
0.00 0.000064 8 8 rt_sigprocmask
0.00 0.000034 17 2 select
0.00 0.000021 11 2 read
0.00 0.000016 16 1 sched_getaffinity
0.00 0.000014 14 1 munmap
0.00 0.000014 14 1 execve
0.00 0.000013 13 1 arch_prctl
0.00 0.000011 11 1 close
0.00 0.000000 0 2 sigaltstack
0.00 0.000000 0 1 gettid
------ ----------- ----------- --------- --------- ----------------
100.00 12.647666 195700 228 total
您可以在for {
<- time.After(time.Second)
if file, ok := <- rotate.File; ok{
中添加fileinfo，在每个循环中，您可以先在结构中检查，然后只执行FileStruct
这是添加stat
<- time.After(time.Second)结论
对于没有% time seconds usecs/call calls errors syscall
------ ----------- ----------- --------- --------- ----------------
65.65 0.001512 35 43 1 futex
23.71 0.000546 5 114 rt_sigaction
3.04 0.000070 9 8 mmap
2.43 0.000056 19 3 clone
2.26 0.000052 7 8 rt_sigprocmask
0.56 0.000013 7 2 stat
0.48 0.000011 11 1 munmap
0.48 0.000011 6 2 sigaltstack
0.43 0.000010 10 1 execve
0.39 0.000009 9 1 sched_getaffinity
0.35 0.000008 8 1 arch_prctl
0.22 0.000005 5 1 gettid
0.00 0.000000 0 2 read
0.00 0.000000 0 3 write
0.00 0.000000 0 1 close
0.00 0.000000 0 1 openat
------ ----------- ----------- --------- --------- ----------------
100.00 0.002303 192 1 total
的相同持续时间代码进行了195K系统调用，其中time.After()只进行了192次系统调用。您可以通过添加已提取的文件信息作为time.After(time.Second)
- 我写了这段代码，但我无法理解我的错误
- 我无法从一个代码实例的列表中删除 None 值，但我可以在另一个实例中。为什么它适用于一个细分市场而不适用于另一个细分市场？
- 是否有可能使 loadstring 不可能等于打印？卢阿
- java中的random.expovariate()
- Appscript 通过会议在 Google 日历中发送电子邮件和创建活动
- 为什么我的 Onclick 箭头功能在 React 中不起作用？
- 在此代码中是否有使用“this”的替代方法？
- 在 SQL Server 和 PostgreSQL 上查询，我如何从第一个表获得第二个表的可视化
- 每千个数字得到
- 更新了城市边界 KML 文件的来源？