使用Java进行TCP网络摄像头流传输

Question

感谢阅读，提前抱歉我的英语不好。

我正在使用OpenCV进行网络摄像头流传输的工作。我的最终目标是制作一个类似Skype的应用程序，所以我首先尝试基本的1:1 TCP模型。

关于TCP 1:1模型，连接后，客户端发送其实时网络摄像头帧，服务器接收并在其jpanel上显示它。

到目前为止，我已经成功接收并在jpanel上显示了一张图片。现在我正在尝试接收连续的帧。起初，问题是服务器端的套接字似乎要等待客户端输入完成，即，它永远不会停止，因为实时帧不断发送。因此，我在发送帧之前先发送每个帧的大小以避免无法停止等待。但这并不有效。客户端继续发送帧，但服务器却无法很好地接收它。例如，如果客户端发送大约25k字节大小的帧，则服务器每次只能读取1到3个字节，即使缓冲区大小为512。

ClientThread.java

package client;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;

import javax.imageio.ImageIO;

import video.VideoCap;

public class ClientThread extends Thread
{

    String   serverIp;
    int      serverPort;
    Socket   socket;
    VideoCap videoCap;

    public ClientThread(Socket socket, String serverIp, int serverPort, VideoCap videoCap)
    {
        this.socket = socket;
        this.serverIp = serverIp;
        this.serverPort = serverPort;
        this.videoCap = videoCap;
    }

    public void run()
    {
        while (ClientUI.calling)
        {
            try
            {
                InputStream in = socket.getInputStream();
                DataInputStream dis = new DataInputStream(in);
                OutputStream out = socket.getOutputStream();
                DataOutputStream dos = new DataOutputStream(out);

                // receive
                int bufSize = dis.readInt();
                while (ClientUI.calling)
                {
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    ImageIO.write(videoCap.getOneFrame(), "jpg", baos);
                    InputStream inputImage = new ByteArrayInputStream(baos.toByteArray());

                    // frame size
                    dos.writeInt(baos.size());
                    out(inputImage, baos, bufSize);
                    Thread.sleep(5000);
                }

            }
            catch (IOException | InterruptedException e)
            {
                e.printStackTrace();
            }
        }
    }

    void out(InputStream in, OutputStream out, int bufSize)
    {
        long size = 0;
        try
        {
            byte[] buf = new byte[bufSize];
            int n;
            while ((n = in.read(buf)) > 0)
            {
                out.write(buf, 0, n);
                size += n;
                System.out.println("size: " + size);
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }

        finally
        {
            System.out.println(getClass().getName() + " :: out >>> sent size: " + size);
        }
    }
}

ServerThread.java

package server;

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;

import javax.imageio.ImageIO;
import javax.swing.JPanel;

public class ServerThread extends Thread
{
    ServerSocket serverSocket;
    Socket       socket;
    JPanel       panel;
    byte[]       buf;

    public ServerThread(ServerSocket serverSocket, JPanel panel, int bufSize)
    {
        this.serverSocket = serverSocket;
        this.panel = panel;
        buf = new byte[bufSize];
    }

    public void run()
    {
        try
        {
            System.out.println("waiting for client");
            socket = serverSocket.accept();
            System.out.println("client accepted");
            InputStream in = socket.getInputStream();
            DataInputStream dis = new DataInputStream(in);
            OutputStream out = socket.getOutputStream();
            DataOutputStream dos = new DataOutputStream(out);
            dos.writeInt(buf.length);

            while (ServerUI.calling)
            {
                int frameSize = dis.readInt();
                ByteArrayOutputStream outImage = new ByteArrayOutputStream();
                long size = 0;
                int n;

                while (frameSize >= size)
                {
                    n = dis.read(buf);
                    if (n == -1)
                        break;
                    outImage.write(buf, 0, n);
                    size += n;
                    System.out.println(n);

                }

                InputStream inputImage = new ByteArrayInputStream(outImage.toByteArray());
                BufferedImage bufferedImage = ImageIO.read(inputImage);
                panel.getGraphics().drawImage(bufferedImage, 0, 0, null);
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
    }

}

- Bana

---

服务器为什么要将bufsize发送给客户端？ - Steve Smith

@SteveSmith 我之前认为如果缓冲区大小一样，发送和接收数据的量也会一样。 - Bana

Answer 1

TCP传输层是可靠的，不会发生数据包丢失，因此可能会出现延迟。您可以使用DatagramSocket代替ServerSocket和Socket通过UDP协议进行传输。

使用DatagramSocket的示例服务器和客户端：

import java.io.IOException; 
import java.net.DatagramPacket; 
import java.net.DatagramSocket; 
public class smallserver { 

public static void main(String[] args) throws IOException { 

    string host = "127.0.0.1";
    int port = 5252;
    byte buf[] = new byte[2]; 
    byte send[] = { 13, 18 };

    //server
    DatagramSocket serverSocket  = new DatagramSocket(port);  
    DatagramPacket receivePacket = new DatagramPacket(buf, 2); 
    serverSocket.receive(receivePacket); 

    // client
    DatagramSocket clientSocket  = new DatagramSocket(host, port); 
    DatagramPacket sendPacket = new DatagramPacket(send, 2, clientSocket.getAddress(), clientSocket.getPort()); 
    clientSocket.send(sendPacket); 
 } 

} 

- Ehsan Jelodar

Answer 2

我将DataOutput/InputStream更改为ObjectOutput/InputStream。 我不确定为什么它没有很好地运行，但我猜这是由于序列化问题。但是byte不一定需要被序列化，所以我不确定。

我会提供可正常工作的代码。由于AudioServer，我将其分成了两个线程，因此上面的代码和下面的代码相当不同。

VideoServerThread.java

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.net.ServerSocket;
import java.net.Socket;
import javax.imageio.ImageIO;
import javax.swing.JPanel;

public class VideoServerThread extends Thread
{
    private ServerSocket serverSocket;
    int                  videoServerPort;
    private Socket       socket;
    private JPanel       panel;
    private boolean      calling;

    public VideoServerThread(ServerSocket serverSocket, int videoServerPort, JPanel panel, boolean calling)
    {
        this.serverSocket = serverSocket;
        this.videoServerPort = videoServerPort;
        this.panel = panel;
        this.calling = calling;
    }

    @Override
    public void run()
    {
        System.out.println("Video Server opened!");
        try
        {
            serverSocket = new ServerSocket(videoServerPort);
            socket = serverSocket.accept();
            InputStream in = socket.getInputStream();
            ObjectInputStream ois = new ObjectInputStream(in);
            BufferedImage bufferedImage;
            InputStream inputImage;
            Frame f;
            while (calling)
            {
                f = (Frame) ois.readObject();
                inputImage = new ByteArrayInputStream(f.bytes);
                bufferedImage = ImageIO.read(inputImage);
                panel.getGraphics().drawImage(bufferedImage, 0, 0, panel.getWidth(), panel.getHeight(), null);
                panel.getGraphics().drawImage(bufferedImage, 0, 0, null);
                bufferedImage.flush();
                inputImage.close();
                f = null;
            }

        }

        catch (IOException e)
        {
            e.printStackTrace();
        }
        catch (ClassNotFoundException e)
        {
            e.printStackTrace();
        }
    }

    class Frame implements Serializable
    {
        public byte[] bytes;
        public Frame(byte[] bytes)
        { 
            this.bytes = bytes;
        }

        public int size()
        {
            return bytes.length;
        }
    }
}

VideoClientThread.java

import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.net.Socket;
import javax.imageio.ImageIO;
import common.Frame;
import video.VideoCap;

public class VideoClientThread extends Thread
{
    private final String formatType = "jpg";
    private VideoCap     videoCap;
    private Socket       socket;
    private String       ip;
    private int          port;
    private boolean      calling;

    public VideoClientThread(VideoCap videoCap, Socket socket, String ip, int port, boolean calling)
    {
        this.videoCap = videoCap;
        this.socket = socket;
        this.ip = ip;
        this.port = port;
        this.calling = calling;
    }

    public void run()
    {
        try
        {
            socket = new Socket(ip, port);
            socket.setSoTimeout(5000);
            ObjectOutputStream oos = new ObjectOutputStream(socket.getOutputStream());
            Frame f;
            BufferedImage bufferedImage;
            while (calling)
            {
                ByteArrayOutputStream fbaos = new ByteArrayOutputStream();
                bufferedImage = videoCap.getOneFrame();
                ImageIO.write(bufferedImage, formatType, fbaos);
                f = new Frame(fbaos.toByteArray());
                oos.writeObject(f);
                oos.flush();
                bufferedImage.flush();
                // Thread.sleep(33);
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        // catch (InterruptedException e)
        // {
        // e.printStackTrace();
        // }
    }
}

- Bana