Develop Reference

Unity3D & Realsense 2.0 D415 - Depth to Terrain

I am trying to generate a runtime terrain from the depth values of the RealSense camera D415.
I got the camera to try and do something for the summer. I never used more than webcams in Unity before so I'm very lost even after going through all the sample scenes.
My terrain is 500 x 500 x 40
using RealSense SDK 2.0
the terrain generator code is below which works with Perlin noise
using UnityEngine;
using Intel.RealSense;
/// <summary>
/// this script is made to test the new terrain in unity with intel realsense depth
/// </summary>
public class TerrainGenerator : MonoBehaviour {
public int depth = 40;
public int width = 500;
public int height = 500;
public float scale = 20f;
public float xOffset = 10;
public float yOffset = 10;
private DepthFrame depthFrame;
public bool scroll;
public float scrollSpeed;
private Pipeline pipe;
private void Start()
{
///
pipe = new Pipeline();
pipe.Start();
///
xOffset = Random.Range(0f, 9999f);
yOffset = Random.Range(0f, 9999f);
scroll = false;
scrollSpeed = 0;
}
void Update()
{
Terrain terrain = GetComponent<Terrain>();
terrain.terrainData = GenerateTerrain(terrain.terrainData);
if(scroll)
xOffset += Time.deltaTime * scrollSpeed;
}
TerrainData GenerateTerrain(TerrainData tData)
{
tData.heightmapResolution = width + 1;
tData.size = new Vector3(width, depth, height);
tData.SetHeights(0, 0, GenerateHeights());
return tData;
}
float[,] GenerateHeights()
{
float[,] heights = new float[width, height];
for (int x = 0; x < width; x++)
{
for (int y = 0; y < height; y++)
{
//heights[x, y] = CalculateHeight(x, y);
var frames = pipe.WaitForFrames();
var depth = frames.DepthFrame;
heights[x, y] = depth.GetDistance(x, y);
}
}
return heights;
}
float CalculateHeight(int x, int y)
{
//float xCoord = (float)x / width * scale + xOffset;
//float yCoord = (float)y / height * scale + yOffset;
//return depthFrame.GetDistance(x,y);
//return Mathf.PerlinNoise(xCoord, yCoord);
using (var frames = pipe.WaitForFrames())
using (var depth = frames.DepthFrame)
return depth.GetDistance(x, y);
}
}
I am looking for some guidance on how to init the camera depth properly. I have never used a camera before with Unity.

Bit off-topic but I wanted to ask how well your above code runs? Are you getting errors such as:
ExternalException: rs2_pipeline_wait_for_frames(pipe:000000006402E3C0)
Rethrow as Exception: Frame didn't arrived within 1000
I'm trying to do something similar and also having problems. Maybe we can work it out together?
Cheers,
Cam

How to determine scale to increase sprite width to screen width

I am trying to achieve something similar to How to increase (animate) the width of the square on both ends in Unity. How can I determine the scale by which to increase the width (of the sprite) for it to fill the whole screen width?
UPDATE
Below is the Swift code for I implemented for expanding the sprite width to take the full screen width:
func expandEnemy () {
spritePosBeforeScaleX = CGPointMake((enemy?.sprite.position.x)!, (enemy?.sprite.anchorPoint.y)!)
enemy?.sprite.anchorPoint = CGPointMake((enemy?.sprite.position.x)! / self.size.width, (enemy?.sprite.anchorPoint.y)!)
let enemyScalingAction = SKAction.scaleXTo(self.size.width / (enemy?.sprite.size.width)!, duration: 1.0)
enemy!.sprite.runAction(enemyScalingAction)
delay(0.1)
{
center = CGPointMake(enemy!.sprite.size.width / 2 - (enemy!.sprite.size.width * enemy!.sprite.anchorPoint.x), enemy!.sprite.size.height / 2 - (enemy!.sprite.size.height * enemy!.sprite.anchorPoint.y))
enemy!.sprite.physicsBody = SKPhysicsBody(rectangleOfSize: enemy!.sprite.size, center: center)
}
}

It all depends on the aspect ratio of the screen and the size of the object with the SpriteRenderer. You need to scale up the gameobject that holds the spriterenderer by a factor where you take these into consideration.
[ExecuteInEditMode]
public class SpriteToScreen : MonoBehaviour {
public float sprw = 256f;
public float sprh = 256f;
float unitspp = 100f;
public float scrw = 0f;
public float scrh = 0f;
public float aspect = 0f;
public float spr_aspect = 1f;
public float factorY = 0.017578125f;
public void Update(){
scrw = Screen.width;
scrh = Screen.height;
aspect = scrw / scrh;
unitspp = this.GetComponent<SpriteRenderer>().sprite.pixelsPerUnit;
sprw = this.GetComponent<SpriteRenderer>().sprite.bounds.size.x * unitspp;
sprh = this.GetComponent<SpriteRenderer>().sprite.bounds.size.y * unitspp;
spr_aspect = sprw / sprh;
this.transform.localScale = new Vector3( (1152f / sprh * aspect) / spr_aspect,
1152f / sprh,
1 );
}
}

You can scale Image of an UI in x-axis full screen. Just get the RectTransform then modify the sizeDelta property of it to the Screen size of the device or the size of the Canvas.
The function below can scale Unity UI Image in x, y or both x and y axis full screen. The Image to scale must be under Canvas. Assign a Sprite to the Source Image of the Image component the code below should work.
//Attach the UI Image to scacle in the Editor here
public GameObject image;
To Scale:
Scale in X-axis Full Screen in 3 seconds:
StartCoroutine(scaleToFullScreen(image, 0, 3f));
Scale in Y-axis Full Screen in 3 seconds:
StartCoroutine(scaleToFullScreen(image, 1, 3f));
Scale in X-axis AND Y-axis Full Screen in 3 seconds:
StartCoroutine(scaleToFullScreen(image, 2, 3f));
The Scale function:
bool isScaling = false;
IEnumerator scaleToFullScreen(GameObject imageToScale, int scaleType, float byTime)
{
if (isScaling)
{
yield break;
}
if (scaleType < 0 || scaleType > 2)
{
Debug.Log("Invalid ScaleType. Valid Scale Types X:0, Y:1, XandY:3");
yield break;
}
isScaling = true;
Canvas canvas = imageToScale.GetComponentInParent<Canvas>();
float x, y;
if (canvas != null)
{
x = canvas.pixelRect.width;
y = canvas.pixelRect.height;
}
else
{
x = Screen.width;
y = Screen.height;
}
RectTransform rect = imageToScale.GetComponent<RectTransform>();
if (rect == null)
{
rect = imageToScale.AddComponent<RectTransform>();
}
//Center the position of the image so that it will be resized equally
rect.anchoredPosition3D = new Vector3(0, 0, rect.anchoredPosition3D.z);
//The default Size
Vector2 originalScale = rect.sizeDelta;
//The new scale we want to scale texture to
Vector2 newScale = originalScale;
if (scaleType == 0)
{
newScale.x = x;
}
else if (scaleType == 1)
{
newScale.y = y;
}
else if (scaleType == 2)
{
newScale.x = x;
newScale.y = y;
}
float counter = 0;
while (counter < byTime)
{
counter += Time.deltaTime;
rect.sizeDelta = Vector2.Lerp(originalScale, newScale, counter / byTime);
yield return null;
}
isScaling = false;
}

XNA Parallaxing background wont show all the layers

Im playing around with the Platformer Starter Kit and so far I've added in horizontal and vertical "camera" movement and Im trying to add inn a parallaxing background. The problem is that after two background layers it stops showing the rest of them. Im very new to XNA and need a little help :). Heres a pic of the problem:
Heres the code. Please tell me if you need some more :)
Layer classes:
class Layer
{
public Texture2D[] Textures { get; private set; }
public float ScrollRate { get; private set; }
public Layer(ContentManager content, string basePath, float scrollRate)
{
// Assumes each layer only has 3 segments.
Textures = new Texture2D[3];
for (int i = 0; i < 3; ++i)
Textures[i] = content.Load<Texture2D>(basePath + "_" + i);
ScrollRate = scrollRate;
}
public void Draw(SpriteBatch spriteBatch, float cameraPosition, float cameraPositionYAxis)
{
// Assume each segment is the same width.
int segmentWidth = Textures[0].Width;
// Calculate which segments to draw and how much to offset them.
float x = cameraPosition * ScrollRate;
float y = ScrollRate;
int leftSegment = (int)Math.Floor(x / segmentWidth);
int rightSegment = leftSegment + 1;
x = (x / segmentWidth - leftSegment) * -segmentWidth;
spriteBatch.Draw(Textures[leftSegment % Textures.Length], new Vector2(x, -y), Color.White);
spriteBatch.Draw(Textures[rightSegment % Textures.Length], new Vector2(x + segmentWidth, -y), Color.White);
}
}
Heres the draw method in my Level.cs with my ScrollCamera (dont know if ScrollCamera has anything to do with it)
public void Draw(GameTime gameTime, SpriteBatch spriteBatch)
{
ScrollCamera(spriteBatch.GraphicsDevice.Viewport);
Matrix cameraTransformYAxis = Matrix.CreateTranslation(-cameraPosition, -cameraPositionYAxis, 0.0f);
spriteBatch.Begin(SpriteSortMode.Immediate, BlendState.AlphaBlend, SamplerState.LinearClamp,
DepthStencilState.Default, RasterizerState.CullCounterClockwise, null, cameraTransformYAxis);
//added this foreach loop
foreach (var layer in layers)
{
layer.Draw(spriteBatch, cameraPosition, cameraPositionYAxis);
}
DrawTiles(spriteBatch);
Player.Draw(gameTime, spriteBatch);
foreach (Enemy enemy in enemies)
{
enemy.Draw(gameTime, spriteBatch);
}
spriteBatch.End();
}
private void ScrollCamera(Viewport viewport)
{
#if ZUNE
const float ViewMargin = 0.4f;
#else
const float ViewMargin = 0.5f;
#endif
float marginWidth = viewport.Width * ViewMargin;
float marginLeft = cameraPosition + marginWidth;
float marginRight = cameraPosition + viewport.Width - marginWidth;
const float TopMargin = 0.4f;
const float BottomMargin = 0.4f;
float marginTop = cameraPositionYAxis + viewport.Height * TopMargin;
float marginBottom = cameraPositionYAxis + viewport.Height - viewport.Height * BottomMargin;
// float maxCameraPositionYOffset = Tile.Height * Height - viewport.Height;
float CameraMovement = 0.0f;
if (Player.Position.X < marginLeft)
CameraMovement = Player.Position.X - marginLeft;
else if (Player.Position.X > marginRight)
CameraMovement = Player.Position.X - marginRight;
//Aktualizuj przesuwanie ekranu, ale zapobiegnij wyjściu poza mape
float maxCameraPosition = Tile.Width * Width - viewport.Width;
cameraPosition = MathHelper.Clamp(cameraPosition + CameraMovement, 0.0f, maxCameraPosition);
float cameraMovementY = 0.0f;
if (Player.Position.Y < marginTop) //above the top margin
cameraMovementY = Player.Position.Y - marginTop;
else if (Player.Position.Y > marginBottom) //below the bottom margin
cameraMovementY = Player.Position.Y - marginBottom;
float maxCameraPositionYOffset = Tile.Height * Height - viewport.Height;
cameraPositionYAxis = MathHelper.Clamp(cameraPositionYAxis + cameraMovementY, 0.0f, maxCameraPositionYOffset);
}
And I think thats it. Please tell me if you need some more code :)

You want to use Linear Wrapping. There's an excellent blog post on it right here. This assumes of course that your texture tiles perfect. You just simply need to to set your linear wrapping mode, code example below:
// Use this one instead!
spriteBatch.Begin(SpriteSortMode.Deferred, null, SamplerState.LinearWrap, null, null);
spriteBatch.Draw(texture, position, new Rectangle(-scrollX, -scrollY, texture.Width, texture.Height), Color.White);
spriteBatch.End();

Light Rendering Results in Low FPS

I am currently developing a class for my XNA game whose rendering the lights on the image. At the time, i have made the source to draw my lightmap, however, the FPS is very low in my source. I know that it is brutally reduced upon looping through each pixel, however, I do not know any other way to get & set each pixel on my Texture in XNA but using the "For" statement?
Current Source:
public struct Light
{
public int Range;
public int Intensity;
public Color LightColor;
public Vector2 LightLocation;
public Light(int _Range, int _Intensity, Color _LightColor, Vector2 _LightLocation)
{
Range = _Range;
Intensity = _Intensity;
LightLocation = _LightLocation;
LightColor = _LightColor;
}
}
public class RenderClass
{
[System.Runtime.InteropServices.DllImport("User32.dll")]
public static extern bool MessageBox(IntPtr h, string S, string C, int a);
public static Texture2D RenderImage(Light[] LightLocations, Texture2D ScreenImage, Viewport v, bool ShadowBack = false)
{
Texture2D[] Images = new Texture2D[LightLocations.Count()];
int curCount = 0;
/*LOOP THROUGHT EACH LIGHT*/
foreach (Light LightLocation in LightLocations)
{
/*VARIABLES*/
Color LightColor = LightLocation.LightColor;
int Range = LightLocation.Range;
int Intensity = LightLocation.Intensity;
/*GET COLORS*/
int Width = v.Width;
int Height = v.Height;
Color[] Data = new Color[Width * Height];
ScreenImage.GetData<Color>(Data);
/*VARIABLES TO SET COLOR*/
Color[] SetColorData = new Color[Width * Height];
/*CIRCEL*/
int Radius = 15 / 2; // Define range to middle [Radius]
int Area = (int)Math.PI * (Radius * Radius);
for (int X = 0; X < Width; X++)
{
for (int Y = 0; Y < Height; Y++)
{
int Destination = X + Y * Width;
#region Light
/*GET COLOR*/
Color nColor = Data[Destination];
/*CREATE NEW COLOR*/
Vector2 MiddlePos = new Vector2(LightLocation.LightLocation.X + Radius, LightLocation.LightLocation.Y + Radius);
Vector2 CurrentLocation = new Vector2(X, Y);
float Distance;
Distance = Vector2.Distance(MiddlePos, CurrentLocation);
Distance *= 100;
Distance /= MathHelper.Clamp(Range, 0, 100);
Vector3 newColors = nColor.ToVector3();
nColor = new Color(
newColors.X,
newColors.Y,
newColors.Z,
Distance / 100);
/*SET COLOR*/
SetColorData[Destination] = nColor; // Add to array
#endregion
#region Shadow
#endregion
}
}
ScreenImage.SetData<Color>(SetColorData);
Images[curCount] = ScreenImage;
curCount++;
}
return Images[0]; // Temporarily returning the first image of the array.
}
}
As you can see, this is a slow and bad method. So I was wondering, is there a better way to get & set each pixel?
Thanks in advance, dotTutorials! =)

I think that job would be best done in a pixel shader.
You could create an Effect file that operates over one light at a time.XNA uses DX9 so you'll be limited to 128 constant registers, which I think you can use to squeeze up to three lights.
So you set your lightmap as a render target, loop through all the lights, set the constant data on your effect, render a render-target-sized quad and in your pixel shader compute your lighting equation.
In essence something like that:
// In LoadContent
RenderTarget2D lightmapRT = new RenderTarget2D(graphics.GraphicsDevice,
128,
128,
false, //No mip-mapping
SurfaceFormat.Color,
DepthFormat.Depth24);
// We now render to the lightmap in Render method
graphics.GraphicsDevice.SetRenderTarget(lightmapRT);
// Lightmap is black by default
graphics.GraphicsDevice.Clear(Color.Black);
// Use the sprite batch to draw quads with custom shader
spriteBatch.Begin(0, BlendState.Opaque, null, null, null, lightmapFx);
foreach (var light in lights)
{
// Pass the light parameters to the shader
lightmapFx.Parameters["Viewport"].SetValue(new Vector2(GraphicsDevice.Viewport.Width, GraphicsDevice.Viewport.Height));
lightmapFx.Parameters["Range"].SetValue(light.Range);
lightmapFx.Parameters["Intensity"].SetValue(light.Intensity);
lightmapFx.Parameters["LightColor"].SetValue(light.LightColor);
lightmapFx.Parameters["LightLocation"].SetValue(light.LightLocation);
// Render quad
spriteBatch.Draw(...);
}
spriteBatch.End();
And the FX file would look something like that:
float Range;
float Intensity;
float3 LightColor;
float2 LightLocation;
float2 Viewport;
struct VStoPS
{
float4 Position : POSITION0;
float2 TexCoord : TEXCOORD0;
};
VStoPS VS(in float4 color : COLOR0,
in float2 texCoord : TEXCOORD0,
in float4 position : POSITION0)
{
VStoPS vsout = (VStoPS)0;
// Half pixel offset for correct texel centering.
vsout.Position.xy -= 0.5;
// Viewport adjustment.
vsout.Position.xy = position.xy / Viewport;
vsout.Position.xy *= float2(2, -2);
vsout.Position.xy -= float2(1, -1);
// Pass texcoords as is
vsout.TexCoord = texCoord;
return vsout;
}
float4 PS(VStoPS psin)
{
// Do calculations here
// Here I just set it to white
return float4(1.0f, 1.0f, 1.0f, 1.0f);
}
technique Main
{
pass p0
{
VertexShader = compile vs_3_0 VS();
PixelShader = compile ps_3_0 PS();
}
}
Note that this non-tested code and probably full of errors. I leave it up to you to figure out what needs to go in the pixel shader.

Drawing a textured quad using XNA

I'm attempting to render a textured quad using the example located here.
I can successfully render the quad, but the texture information appears to be lost. The quad takes the color of the underlying texture, though.
I've checked the obvious problems ("Does the BasicEffect rendering the quad have the TextureEnabled property set to true?") and can't immediately see the problem.
Code below:
public class Quad
{
public VertexPositionNormalTexture[] Vertices;
public Vector3 Origin;
public Vector3 Up;
public Vector3 Normal;
public Vector3 Left;
public Vector3 UpperLeft;
public Vector3 UpperRight;
public Vector3 LowerLeft;
public Vector3 LowerRight;
public int[] Indexes;
public Quad(Vector3 origin, Vector3 normal, Vector3 up,
float width, float height)
{
this.Vertices = new VertexPositionNormalTexture[4];
this.Indexes = new int[6];
this.Origin = origin;
this.Normal = normal;
this.Up = up;
// Calculate the quad corners
this.Left = Vector3.Cross(normal, this.Up);
Vector3 uppercenter = (this.Up * height / 2) + origin;
this.UpperLeft = uppercenter + (this.Left * width / 2);
this.UpperRight = uppercenter - (this.Left * width / 2);
this.LowerLeft = this.UpperLeft - (this.Up * height);
this.LowerRight = this.UpperRight - (this.Up * height);
this.FillVertices();
}
private void FillVertices()
{
Vector2 textureUpperLeft = new Vector2(0.0f, 0.0f);
Vector2 textureUpperRight = new Vector2(1.0f, 0.0f);
Vector2 textureLowerLeft = new Vector2(0.0f, 1.0f);
Vector2 textureLowerRight = new Vector2(1.0f, 1.0f);
for (int i = 0; i < this.Vertices.Length; i++)
{
this.Vertices[i].Normal = this.Normal;
}
this.Vertices[0].Position = this.LowerLeft;
this.Vertices[0].TextureCoordinate = textureLowerLeft;
this.Vertices[1].Position = this.UpperLeft;
this.Vertices[1].TextureCoordinate = textureUpperLeft;
this.Vertices[2].Position = this.LowerRight;
this.Vertices[2].TextureCoordinate = textureLowerRight;
this.Vertices[3].Position = this.UpperRight;
this.Vertices[3].TextureCoordinate = textureUpperRight;
this.Indexes[0] = 0;
this.Indexes[1] = 1;
this.Indexes[2] = 2;
this.Indexes[3] = 2;
this.Indexes[4] = 1;
this.Indexes[5] = 3;
}
}
this.quadEffect = new BasicEffect(this.GraphicsDevice, null);
this.quadEffect.AmbientLightColor = new Vector3(0.8f, 0.8f, 0.8f);
this.quadEffect.LightingEnabled = true;
this.quadEffect.World = Matrix.Identity;
this.quadEffect.View = this.View;
this.quadEffect.Projection = this.Projection;
this.quadEffect.TextureEnabled = true;
this.quadEffect.Texture = someTexture;
this.quad = new Quad(Vector3.Zero, Vector3.UnitZ, Vector3.Up, 2, 2);
this.quadVertexDecl = new VertexDeclaration(this.GraphicsDevice, VertexPositionNormalTexture.VertexElements);
public override void Draw(GameTime gameTime)
{
this.GraphicsDevice.Textures[0] = this.SpriteDictionary["B1S1I800"];
this.GraphicsDevice.VertexDeclaration = quadVertexDecl;
quadEffect.Begin();
foreach (EffectPass pass in quadEffect.CurrentTechnique.Passes)
{
pass.Begin();
GraphicsDevice.DrawUserIndexedPrimitives<VertexPositionNormalTexture>(
PrimitiveType.TriangleList,
beamQuad.Vertices, 0, 4,
beamQuad.Indexes, 0, 2);
pass.End();
}
quadEffect.End();
}

From what I can see, this should work. The only thing I can imagine, which isn't in this code, is that the loading of the texture goes wrong somewhere. I also can't quite visualize what you mean that the quad has the underlying color of the texture? Do you have a screenshot for us?
Also, if something does show up, a very distorted version of your texture for example, it could be possible that the rendering of other stuff has effect on the rendering of the quad. For example if you draw the quad while the graphicsdevice has another vertex declaration on it, or if the previous thing rendered set some exotic rendering state, or if you're drawing the quad within the drawing code of something else. Try isolating this code, into a fresh project or something, or disable the rendering of everything else.